@sparkleideas/plugins 3.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +401 -0
- package/__tests__/collection-manager.test.ts +332 -0
- package/__tests__/dependency-graph.test.ts +434 -0
- package/__tests__/enhanced-plugin-registry.test.ts +488 -0
- package/__tests__/plugin-registry.test.ts +368 -0
- package/__tests__/ruvector-bridge.test.ts +2429 -0
- package/__tests__/ruvector-integration.test.ts +1602 -0
- package/__tests__/ruvector-migrations.test.ts +1099 -0
- package/__tests__/ruvector-quantization.test.ts +846 -0
- package/__tests__/ruvector-streaming.test.ts +1088 -0
- package/__tests__/sdk.test.ts +325 -0
- package/__tests__/security.test.ts +348 -0
- package/__tests__/utils/ruvector-test-utils.ts +860 -0
- package/examples/plugin-creator/index.ts +636 -0
- package/examples/plugin-creator/plugin-creator.test.ts +312 -0
- package/examples/ruvector/README.md +288 -0
- package/examples/ruvector/attention-patterns.ts +394 -0
- package/examples/ruvector/basic-usage.ts +288 -0
- package/examples/ruvector/docker-compose.yml +75 -0
- package/examples/ruvector/gnn-analysis.ts +501 -0
- package/examples/ruvector/hyperbolic-hierarchies.ts +557 -0
- package/examples/ruvector/init-db.sql +119 -0
- package/examples/ruvector/quantization.ts +680 -0
- package/examples/ruvector/self-learning.ts +447 -0
- package/examples/ruvector/semantic-search.ts +576 -0
- package/examples/ruvector/streaming-large-data.ts +507 -0
- package/examples/ruvector/transactions.ts +594 -0
- package/examples/ruvector-plugins/hook-pattern-library.ts +486 -0
- package/examples/ruvector-plugins/index.ts +79 -0
- package/examples/ruvector-plugins/intent-router.ts +354 -0
- package/examples/ruvector-plugins/mcp-tool-optimizer.ts +424 -0
- package/examples/ruvector-plugins/reasoning-bank.ts +657 -0
- package/examples/ruvector-plugins/ruvector-plugins.test.ts +518 -0
- package/examples/ruvector-plugins/semantic-code-search.ts +498 -0
- package/examples/ruvector-plugins/shared/index.ts +20 -0
- package/examples/ruvector-plugins/shared/vector-utils.ts +257 -0
- package/examples/ruvector-plugins/sona-learning.ts +445 -0
- package/package.json +97 -0
- package/src/collections/collection-manager.ts +661 -0
- package/src/collections/index.ts +56 -0
- package/src/collections/official/index.ts +1040 -0
- package/src/core/base-plugin.ts +416 -0
- package/src/core/plugin-interface.ts +215 -0
- package/src/hooks/index.ts +685 -0
- package/src/index.ts +378 -0
- package/src/integrations/agentic-flow.ts +743 -0
- package/src/integrations/index.ts +88 -0
- package/src/integrations/ruvector/ARCHITECTURE.md +1245 -0
- package/src/integrations/ruvector/attention-advanced.ts +1040 -0
- package/src/integrations/ruvector/attention-executor.ts +782 -0
- package/src/integrations/ruvector/attention-mechanisms.ts +757 -0
- package/src/integrations/ruvector/attention.ts +1063 -0
- package/src/integrations/ruvector/gnn.ts +3050 -0
- package/src/integrations/ruvector/hyperbolic.ts +1948 -0
- package/src/integrations/ruvector/index.ts +394 -0
- package/src/integrations/ruvector/migrations/001_create_extension.sql +135 -0
- package/src/integrations/ruvector/migrations/002_create_vector_tables.sql +259 -0
- package/src/integrations/ruvector/migrations/003_create_indices.sql +328 -0
- package/src/integrations/ruvector/migrations/004_create_functions.sql +598 -0
- package/src/integrations/ruvector/migrations/005_create_attention_functions.sql +654 -0
- package/src/integrations/ruvector/migrations/006_create_gnn_functions.sql +728 -0
- package/src/integrations/ruvector/migrations/007_create_hyperbolic_functions.sql +762 -0
- package/src/integrations/ruvector/migrations/index.ts +35 -0
- package/src/integrations/ruvector/migrations/migrations.ts +647 -0
- package/src/integrations/ruvector/quantization.ts +2036 -0
- package/src/integrations/ruvector/ruvector-bridge.ts +2000 -0
- package/src/integrations/ruvector/self-learning.ts +2376 -0
- package/src/integrations/ruvector/streaming.ts +1737 -0
- package/src/integrations/ruvector/types.ts +1945 -0
- package/src/providers/index.ts +643 -0
- package/src/registry/dependency-graph.ts +568 -0
- package/src/registry/enhanced-plugin-registry.ts +994 -0
- package/src/registry/plugin-registry.ts +604 -0
- package/src/sdk/index.ts +563 -0
- package/src/security/index.ts +594 -0
- package/src/types/index.ts +446 -0
- package/src/workers/index.ts +700 -0
- package/tmp.json +0 -0
- package/tsconfig.json +25 -0
- package/vitest.config.ts +23 -0
|
@@ -0,0 +1,2376 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RuVector Self-Learning Optimization Module
|
|
3
|
+
*
|
|
4
|
+
* SONA-inspired self-learning features for the RuVector PostgreSQL Bridge.
|
|
5
|
+
* Implements adaptive query optimization, intelligent index tuning,
|
|
6
|
+
* pattern recognition, and continuous learning with EWC++ protection.
|
|
7
|
+
*
|
|
8
|
+
* @module @sparkleideas/plugins/integrations/ruvector/self-learning
|
|
9
|
+
* @version 1.0.0
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type {
|
|
13
|
+
VectorIndexType,
|
|
14
|
+
DistanceMetric,
|
|
15
|
+
IndexStats,
|
|
16
|
+
QueryStats,
|
|
17
|
+
VectorSearchOptions,
|
|
18
|
+
} from './types.js';
|
|
19
|
+
|
|
20
|
+
// ============================================================================
|
|
21
|
+
// Query Analysis Types
|
|
22
|
+
// ============================================================================
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Analysis result for a SQL query.
|
|
26
|
+
*/
|
|
27
|
+
export interface QueryAnalysis {
|
|
28
|
+
/** Original SQL query */
|
|
29
|
+
readonly sql: string;
|
|
30
|
+
/** Query type (SELECT, INSERT, UPDATE, DELETE) */
|
|
31
|
+
readonly queryType: QueryType;
|
|
32
|
+
/** Tables referenced in the query */
|
|
33
|
+
readonly tables: string[];
|
|
34
|
+
/** Columns referenced in the query */
|
|
35
|
+
readonly columns: string[];
|
|
36
|
+
/** Vector operations detected */
|
|
37
|
+
readonly vectorOperations: VectorOperation[];
|
|
38
|
+
/** Estimated complexity score (0-1) */
|
|
39
|
+
readonly complexity: number;
|
|
40
|
+
/** Index usage hints */
|
|
41
|
+
readonly indexHints: IndexHint[];
|
|
42
|
+
/** Potential bottlenecks */
|
|
43
|
+
readonly bottlenecks: Bottleneck[];
|
|
44
|
+
/** Parse time in milliseconds */
|
|
45
|
+
readonly parseTimeMs: number;
|
|
46
|
+
/** Query fingerprint for deduplication */
|
|
47
|
+
readonly fingerprint: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Query types supported.
|
|
52
|
+
*/
|
|
53
|
+
export type QueryType = 'SELECT' | 'INSERT' | 'UPDATE' | 'DELETE' | 'UNKNOWN';
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Vector operation in a query.
|
|
57
|
+
*/
|
|
58
|
+
export interface VectorOperation {
|
|
59
|
+
/** Operation type */
|
|
60
|
+
readonly type: 'search' | 'insert' | 'update' | 'aggregate' | 'distance';
|
|
61
|
+
/** Table name */
|
|
62
|
+
readonly table: string;
|
|
63
|
+
/** Column name */
|
|
64
|
+
readonly column: string;
|
|
65
|
+
/** Distance metric used */
|
|
66
|
+
readonly metric?: DistanceMetric;
|
|
67
|
+
/** K value for KNN */
|
|
68
|
+
readonly k?: number;
|
|
69
|
+
/** Estimated cost */
|
|
70
|
+
readonly estimatedCost: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Index usage hint.
|
|
75
|
+
*/
|
|
76
|
+
export interface IndexHint {
|
|
77
|
+
/** Recommended index type */
|
|
78
|
+
readonly indexType: VectorIndexType;
|
|
79
|
+
/** Table name */
|
|
80
|
+
readonly table: string;
|
|
81
|
+
/** Column name */
|
|
82
|
+
readonly column: string;
|
|
83
|
+
/** Confidence score (0-1) */
|
|
84
|
+
readonly confidence: number;
|
|
85
|
+
/** Expected speedup factor */
|
|
86
|
+
readonly expectedSpeedup: number;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Query bottleneck.
|
|
91
|
+
*/
|
|
92
|
+
export interface Bottleneck {
|
|
93
|
+
/** Bottleneck type */
|
|
94
|
+
readonly type: 'full_scan' | 'missing_index' | 'cartesian_product' | 'large_sort' | 'expensive_function';
|
|
95
|
+
/** Description */
|
|
96
|
+
readonly description: string;
|
|
97
|
+
/** Severity (1-10) */
|
|
98
|
+
readonly severity: number;
|
|
99
|
+
/** Suggested fix */
|
|
100
|
+
readonly suggestion: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Query optimization suggestion.
|
|
105
|
+
*/
|
|
106
|
+
export interface Optimization {
|
|
107
|
+
/** Optimization type */
|
|
108
|
+
readonly type: OptimizationType;
|
|
109
|
+
/** Description of the optimization */
|
|
110
|
+
readonly description: string;
|
|
111
|
+
/** Original query fragment */
|
|
112
|
+
readonly original: string;
|
|
113
|
+
/** Optimized query fragment */
|
|
114
|
+
readonly optimized: string;
|
|
115
|
+
/** Expected improvement percentage */
|
|
116
|
+
readonly expectedImprovement: number;
|
|
117
|
+
/** Confidence score (0-1) */
|
|
118
|
+
readonly confidence: number;
|
|
119
|
+
/** Risk level */
|
|
120
|
+
readonly risk: 'low' | 'medium' | 'high';
|
|
121
|
+
/** Apply automatically */
|
|
122
|
+
readonly autoApply: boolean;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Types of query optimizations.
|
|
127
|
+
*/
|
|
128
|
+
export type OptimizationType =
|
|
129
|
+
| 'index_usage'
|
|
130
|
+
| 'query_rewrite'
|
|
131
|
+
| 'parameter_tuning'
|
|
132
|
+
| 'caching'
|
|
133
|
+
| 'batching'
|
|
134
|
+
| 'projection_pushdown'
|
|
135
|
+
| 'filter_pushdown'
|
|
136
|
+
| 'limit_pushdown'
|
|
137
|
+
| 'parallel_execution';
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Query execution statistics.
|
|
141
|
+
*/
|
|
142
|
+
export interface QueryExecutionStats {
|
|
143
|
+
/** Query fingerprint */
|
|
144
|
+
readonly fingerprint: string;
|
|
145
|
+
/** SQL query */
|
|
146
|
+
readonly sql: string;
|
|
147
|
+
/** Execution count */
|
|
148
|
+
readonly executionCount: number;
|
|
149
|
+
/** Total execution time (ms) */
|
|
150
|
+
readonly totalDurationMs: number;
|
|
151
|
+
/** Average execution time (ms) */
|
|
152
|
+
readonly avgDurationMs: number;
|
|
153
|
+
/** Min execution time (ms) */
|
|
154
|
+
readonly minDurationMs: number;
|
|
155
|
+
/** Max execution time (ms) */
|
|
156
|
+
readonly maxDurationMs: number;
|
|
157
|
+
/** P95 execution time (ms) */
|
|
158
|
+
readonly p95DurationMs: number;
|
|
159
|
+
/** P99 execution time (ms) */
|
|
160
|
+
readonly p99DurationMs: number;
|
|
161
|
+
/** Total rows returned */
|
|
162
|
+
readonly totalRows: number;
|
|
163
|
+
/** Average rows per execution */
|
|
164
|
+
readonly avgRows: number;
|
|
165
|
+
/** Last executed timestamp */
|
|
166
|
+
readonly lastExecuted: Date;
|
|
167
|
+
/** First executed timestamp */
|
|
168
|
+
readonly firstExecuted: Date;
|
|
169
|
+
/** Error count */
|
|
170
|
+
readonly errorCount: number;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ============================================================================
|
|
174
|
+
// Index Tuning Types
|
|
175
|
+
// ============================================================================
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Workload analysis result.
|
|
179
|
+
*/
|
|
180
|
+
export interface WorkloadAnalysis {
|
|
181
|
+
/** Analysis timestamp */
|
|
182
|
+
readonly timestamp: Date;
|
|
183
|
+
/** Analysis duration (ms) */
|
|
184
|
+
readonly durationMs: number;
|
|
185
|
+
/** Total queries analyzed */
|
|
186
|
+
readonly totalQueries: number;
|
|
187
|
+
/** Query type distribution */
|
|
188
|
+
readonly queryDistribution: Map<QueryType, number>;
|
|
189
|
+
/** Most frequent query patterns */
|
|
190
|
+
readonly topPatterns: QueryPattern[];
|
|
191
|
+
/** Hot tables (most accessed) */
|
|
192
|
+
readonly hotTables: TableAccess[];
|
|
193
|
+
/** Index usage summary */
|
|
194
|
+
readonly indexUsage: IndexUsageSummary[];
|
|
195
|
+
/** Workload characteristics */
|
|
196
|
+
readonly characteristics: WorkloadCharacteristics;
|
|
197
|
+
/** Recommendations */
|
|
198
|
+
readonly recommendations: WorkloadRecommendation[];
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Query pattern from workload analysis.
|
|
203
|
+
*/
|
|
204
|
+
export interface QueryPattern {
|
|
205
|
+
/** Pattern fingerprint */
|
|
206
|
+
readonly fingerprint: string;
|
|
207
|
+
/** Example query */
|
|
208
|
+
readonly example: string;
|
|
209
|
+
/** Execution frequency */
|
|
210
|
+
readonly frequency: number;
|
|
211
|
+
/** Average duration (ms) */
|
|
212
|
+
readonly avgDurationMs: number;
|
|
213
|
+
/** Tables involved */
|
|
214
|
+
readonly tables: string[];
|
|
215
|
+
/** Is vector search */
|
|
216
|
+
readonly isVectorSearch: boolean;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Table access statistics.
|
|
221
|
+
*/
|
|
222
|
+
export interface TableAccess {
|
|
223
|
+
/** Table name */
|
|
224
|
+
readonly tableName: string;
|
|
225
|
+
/** Read count */
|
|
226
|
+
readonly reads: number;
|
|
227
|
+
/** Write count */
|
|
228
|
+
readonly writes: number;
|
|
229
|
+
/** Vector search count */
|
|
230
|
+
readonly vectorSearches: number;
|
|
231
|
+
/** Average scan size */
|
|
232
|
+
readonly avgScanSize: number;
|
|
233
|
+
/** Is frequently accessed */
|
|
234
|
+
readonly isHot: boolean;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Index usage summary.
|
|
239
|
+
*/
|
|
240
|
+
export interface IndexUsageSummary {
|
|
241
|
+
/** Index name */
|
|
242
|
+
readonly indexName: string;
|
|
243
|
+
/** Table name */
|
|
244
|
+
readonly tableName: string;
|
|
245
|
+
/** Index type */
|
|
246
|
+
readonly indexType: VectorIndexType;
|
|
247
|
+
/** Scan count */
|
|
248
|
+
readonly scanCount: number;
|
|
249
|
+
/** Tuple reads */
|
|
250
|
+
readonly tupleReads: number;
|
|
251
|
+
/** Tuple fetches */
|
|
252
|
+
readonly tupleFetches: number;
|
|
253
|
+
/** Is underutilized */
|
|
254
|
+
readonly isUnderutilized: boolean;
|
|
255
|
+
/** Recommendation */
|
|
256
|
+
readonly recommendation: 'keep' | 'drop' | 'rebuild' | 'tune';
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Workload characteristics.
|
|
261
|
+
*/
|
|
262
|
+
export interface WorkloadCharacteristics {
|
|
263
|
+
/** Read/write ratio */
|
|
264
|
+
readonly readWriteRatio: number;
|
|
265
|
+
/** Vector search percentage */
|
|
266
|
+
readonly vectorSearchPercentage: number;
|
|
267
|
+
/** Average query complexity */
|
|
268
|
+
readonly avgComplexity: number;
|
|
269
|
+
/** Peak hours (0-23) */
|
|
270
|
+
readonly peakHours: number[];
|
|
271
|
+
/** Is OLTP-like */
|
|
272
|
+
readonly isOLTP: boolean;
|
|
273
|
+
/** Is OLAP-like */
|
|
274
|
+
readonly isOLAP: boolean;
|
|
275
|
+
/** Is hybrid */
|
|
276
|
+
readonly isHybrid: boolean;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Workload-based recommendation.
|
|
281
|
+
*/
|
|
282
|
+
export interface WorkloadRecommendation {
|
|
283
|
+
/** Recommendation type */
|
|
284
|
+
readonly type: 'create_index' | 'drop_index' | 'tune_parameter' | 'partition_table' | 'materialize_view';
|
|
285
|
+
/** Priority (1-10) */
|
|
286
|
+
readonly priority: number;
|
|
287
|
+
/** Description */
|
|
288
|
+
readonly description: string;
|
|
289
|
+
/** Estimated impact */
|
|
290
|
+
readonly estimatedImpact: string;
|
|
291
|
+
/** SQL to execute */
|
|
292
|
+
readonly sql?: string;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Index suggestion.
|
|
297
|
+
*/
|
|
298
|
+
export interface IndexSuggestion {
|
|
299
|
+
/** Table name */
|
|
300
|
+
readonly tableName: string;
|
|
301
|
+
/** Column name */
|
|
302
|
+
readonly columnName: string;
|
|
303
|
+
/** Suggested index type */
|
|
304
|
+
readonly indexType: VectorIndexType;
|
|
305
|
+
/** Suggested index name */
|
|
306
|
+
readonly indexName: string;
|
|
307
|
+
/** Distance metric */
|
|
308
|
+
readonly metric?: DistanceMetric;
|
|
309
|
+
/** HNSW M parameter */
|
|
310
|
+
readonly m?: number;
|
|
311
|
+
/** HNSW ef_construction */
|
|
312
|
+
readonly efConstruction?: number;
|
|
313
|
+
/** IVF lists */
|
|
314
|
+
readonly lists?: number;
|
|
315
|
+
/** Confidence score (0-1) */
|
|
316
|
+
readonly confidence: number;
|
|
317
|
+
/** Expected improvement */
|
|
318
|
+
readonly expectedImprovement: number;
|
|
319
|
+
/** Rationale */
|
|
320
|
+
readonly rationale: string;
|
|
321
|
+
/** CREATE INDEX SQL */
|
|
322
|
+
readonly createSql: string;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* HNSW parameters.
|
|
327
|
+
*/
|
|
328
|
+
export interface HNSWParams {
|
|
329
|
+
/** M parameter (connections per layer) */
|
|
330
|
+
readonly m: number;
|
|
331
|
+
/** ef_construction parameter */
|
|
332
|
+
readonly efConstruction: number;
|
|
333
|
+
/** ef_search parameter */
|
|
334
|
+
readonly efSearch: number;
|
|
335
|
+
/** Optimal for workload */
|
|
336
|
+
readonly optimizedFor: 'recall' | 'speed' | 'balanced';
|
|
337
|
+
/** Tuning confidence (0-1) */
|
|
338
|
+
readonly confidence: number;
|
|
339
|
+
/** Estimated recall */
|
|
340
|
+
readonly estimatedRecall: number;
|
|
341
|
+
/** Estimated QPS */
|
|
342
|
+
readonly estimatedQps: number;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// ============================================================================
|
|
346
|
+
// Pattern Recognition Types
|
|
347
|
+
// ============================================================================
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Query history entry.
|
|
351
|
+
*/
|
|
352
|
+
export interface QueryHistory {
|
|
353
|
+
/** Query fingerprint */
|
|
354
|
+
readonly fingerprint: string;
|
|
355
|
+
/** SQL query */
|
|
356
|
+
readonly sql: string;
|
|
357
|
+
/** Execution timestamp */
|
|
358
|
+
readonly timestamp: Date;
|
|
359
|
+
/** Duration (ms) */
|
|
360
|
+
readonly durationMs: number;
|
|
361
|
+
/** Rows returned */
|
|
362
|
+
readonly rowCount: number;
|
|
363
|
+
/** Was successful */
|
|
364
|
+
readonly success: boolean;
|
|
365
|
+
/** User/session ID */
|
|
366
|
+
readonly sessionId?: string;
|
|
367
|
+
/** Context metadata */
|
|
368
|
+
readonly context?: Record<string, unknown>;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Detected query pattern.
|
|
373
|
+
*/
|
|
374
|
+
export interface Pattern {
|
|
375
|
+
/** Pattern ID */
|
|
376
|
+
readonly id: string;
|
|
377
|
+
/** Pattern type */
|
|
378
|
+
readonly type: PatternType;
|
|
379
|
+
/** Pattern signature */
|
|
380
|
+
readonly signature: string;
|
|
381
|
+
/** Description */
|
|
382
|
+
readonly description: string;
|
|
383
|
+
/** Confidence score (0-1) */
|
|
384
|
+
readonly confidence: number;
|
|
385
|
+
/** Occurrence count */
|
|
386
|
+
readonly occurrences: number;
|
|
387
|
+
/** Example queries matching this pattern */
|
|
388
|
+
readonly examples: string[];
|
|
389
|
+
/** Temporal characteristics */
|
|
390
|
+
readonly temporal?: TemporalPattern;
|
|
391
|
+
/** Performance characteristics */
|
|
392
|
+
readonly performance: PerformancePattern;
|
|
393
|
+
/** First detected */
|
|
394
|
+
readonly firstDetected: Date;
|
|
395
|
+
/** Last detected */
|
|
396
|
+
readonly lastDetected: Date;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Pattern types.
|
|
401
|
+
*/
|
|
402
|
+
export type PatternType =
|
|
403
|
+
| 'sequential_access'
|
|
404
|
+
| 'random_access'
|
|
405
|
+
| 'bulk_insert'
|
|
406
|
+
| 'bulk_update'
|
|
407
|
+
| 'similarity_search'
|
|
408
|
+
| 'range_query'
|
|
409
|
+
| 'aggregation'
|
|
410
|
+
| 'join_pattern'
|
|
411
|
+
| 'periodic'
|
|
412
|
+
| 'burst'
|
|
413
|
+
| 'degrading_performance';
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Temporal pattern characteristics.
|
|
417
|
+
*/
|
|
418
|
+
export interface TemporalPattern {
|
|
419
|
+
/** Is periodic */
|
|
420
|
+
readonly isPeriodic: boolean;
|
|
421
|
+
/** Period in seconds (if periodic) */
|
|
422
|
+
readonly periodSeconds?: number;
|
|
423
|
+
/** Peak times (hour of day) */
|
|
424
|
+
readonly peakHours: number[];
|
|
425
|
+
/** Trend direction */
|
|
426
|
+
readonly trend: 'increasing' | 'decreasing' | 'stable' | 'volatile';
|
|
427
|
+
/** Seasonality detected */
|
|
428
|
+
readonly hasSeasonality: boolean;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Performance pattern.
|
|
433
|
+
*/
|
|
434
|
+
export interface PerformancePattern {
|
|
435
|
+
/** Average response time trend */
|
|
436
|
+
readonly responseTrend: 'improving' | 'degrading' | 'stable';
|
|
437
|
+
/** Variance coefficient */
|
|
438
|
+
readonly varianceCoefficient: number;
|
|
439
|
+
/** Has outliers */
|
|
440
|
+
readonly hasOutliers: boolean;
|
|
441
|
+
/** Percentile distribution */
|
|
442
|
+
readonly percentiles: {
|
|
443
|
+
readonly p50: number;
|
|
444
|
+
readonly p75: number;
|
|
445
|
+
readonly p90: number;
|
|
446
|
+
readonly p95: number;
|
|
447
|
+
readonly p99: number;
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* Query prediction context.
|
|
453
|
+
*/
|
|
454
|
+
export interface Context {
|
|
455
|
+
/** Current session ID */
|
|
456
|
+
readonly sessionId?: string;
|
|
457
|
+
/** Recent query fingerprints */
|
|
458
|
+
readonly recentQueries: string[];
|
|
459
|
+
/** Current time */
|
|
460
|
+
readonly timestamp: Date;
|
|
461
|
+
/** User context */
|
|
462
|
+
readonly userContext?: Record<string, unknown>;
|
|
463
|
+
/** Application context */
|
|
464
|
+
readonly appContext?: Record<string, unknown>;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* Query anomaly.
|
|
469
|
+
*/
|
|
470
|
+
export interface Anomaly {
|
|
471
|
+
/** Anomaly ID */
|
|
472
|
+
readonly id: string;
|
|
473
|
+
/** Anomaly type */
|
|
474
|
+
readonly type: AnomalyType;
|
|
475
|
+
/** Affected query */
|
|
476
|
+
readonly query: string;
|
|
477
|
+
/** Query fingerprint */
|
|
478
|
+
readonly fingerprint: string;
|
|
479
|
+
/** Detection timestamp */
|
|
480
|
+
readonly timestamp: Date;
|
|
481
|
+
/** Severity (1-10) */
|
|
482
|
+
readonly severity: number;
|
|
483
|
+
/** Description */
|
|
484
|
+
readonly description: string;
|
|
485
|
+
/** Expected value */
|
|
486
|
+
readonly expected: number;
|
|
487
|
+
/** Actual value */
|
|
488
|
+
readonly actual: number;
|
|
489
|
+
/** Deviation from normal */
|
|
490
|
+
readonly deviation: number;
|
|
491
|
+
/** Possible causes */
|
|
492
|
+
readonly possibleCauses: string[];
|
|
493
|
+
/** Recommended actions */
|
|
494
|
+
readonly recommendations: string[];
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Anomaly types.
|
|
499
|
+
*/
|
|
500
|
+
export type AnomalyType =
|
|
501
|
+
| 'slow_query'
|
|
502
|
+
| 'high_resource_usage'
|
|
503
|
+
| 'unusual_pattern'
|
|
504
|
+
| 'error_spike'
|
|
505
|
+
| 'traffic_anomaly'
|
|
506
|
+
| 'data_drift'
|
|
507
|
+
| 'index_degradation'
|
|
508
|
+
| 'cardinality_change';
|
|
509
|
+
|
|
510
|
+
// ============================================================================
|
|
511
|
+
// Learning System Types
|
|
512
|
+
// ============================================================================
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Learning configuration.
|
|
516
|
+
*/
|
|
517
|
+
export interface LearningConfig {
|
|
518
|
+
/** Enable micro-learning */
|
|
519
|
+
readonly enableMicroLearning: boolean;
|
|
520
|
+
/** Micro-learning threshold (ms) */
|
|
521
|
+
readonly microLearningThresholdMs: number;
|
|
522
|
+
/** Enable background learning */
|
|
523
|
+
readonly enableBackgroundLearning: boolean;
|
|
524
|
+
/** Background learning interval (ms) */
|
|
525
|
+
readonly backgroundLearningIntervalMs: number;
|
|
526
|
+
/** Enable EWC++ */
|
|
527
|
+
readonly enableEWC: boolean;
|
|
528
|
+
/** EWC lambda (regularization strength) */
|
|
529
|
+
readonly ewcLambda: number;
|
|
530
|
+
/** Maximum patterns to retain */
|
|
531
|
+
readonly maxPatterns: number;
|
|
532
|
+
/** Pattern expiry time (ms) */
|
|
533
|
+
readonly patternExpiryMs: number;
|
|
534
|
+
/** Learning rate */
|
|
535
|
+
readonly learningRate: number;
|
|
536
|
+
/** Momentum */
|
|
537
|
+
readonly momentum: number;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Learning statistics.
|
|
542
|
+
*/
|
|
543
|
+
export interface LearningStats {
|
|
544
|
+
/** Total patterns learned */
|
|
545
|
+
readonly totalPatterns: number;
|
|
546
|
+
/** Active patterns */
|
|
547
|
+
readonly activePatterns: number;
|
|
548
|
+
/** Expired patterns */
|
|
549
|
+
readonly expiredPatterns: number;
|
|
550
|
+
/** Micro-learning events */
|
|
551
|
+
readonly microLearningEvents: number;
|
|
552
|
+
/** Background learning cycles */
|
|
553
|
+
readonly backgroundLearningCycles: number;
|
|
554
|
+
/** EWC consolidations */
|
|
555
|
+
readonly ewcConsolidations: number;
|
|
556
|
+
/** Average learning time (ms) */
|
|
557
|
+
readonly avgLearningTimeMs: number;
|
|
558
|
+
/** Memory usage (bytes) */
|
|
559
|
+
readonly memoryUsageBytes: number;
|
|
560
|
+
/** Last learning timestamp */
|
|
561
|
+
readonly lastLearningTimestamp: Date;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* EWC++ state for preventing catastrophic forgetting.
|
|
566
|
+
*/
|
|
567
|
+
export interface EWCState {
|
|
568
|
+
/** Fisher information matrix (diagonal approximation) */
|
|
569
|
+
readonly fisherDiagonal: Map<string, number>;
|
|
570
|
+
/** Previous parameter values */
|
|
571
|
+
readonly previousParams: Map<string, number>;
|
|
572
|
+
/** Consolidation count */
|
|
573
|
+
readonly consolidationCount: number;
|
|
574
|
+
/** Last consolidation timestamp */
|
|
575
|
+
readonly lastConsolidation: Date;
|
|
576
|
+
/** Protected patterns */
|
|
577
|
+
readonly protectedPatterns: Set<string>;
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// ============================================================================
|
|
581
|
+
// Query Optimizer Implementation
|
|
582
|
+
// ============================================================================
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Query Optimizer for analyzing and optimizing SQL queries.
|
|
586
|
+
* Implements SONA-inspired micro-learning for real-time adaptation.
|
|
587
|
+
*/
|
|
588
|
+
export class QueryOptimizer {
|
|
589
|
+
private readonly queryStats: Map<string, QueryExecutionStats> = new Map();
|
|
590
|
+
private readonly optimizationCache: Map<string, Optimization[]> = new Map();
|
|
591
|
+
private readonly config: LearningConfig;
|
|
592
|
+
|
|
593
|
+
constructor(config?: Partial<LearningConfig>) {
|
|
594
|
+
this.config = {
|
|
595
|
+
enableMicroLearning: true,
|
|
596
|
+
microLearningThresholdMs: 0.1, // <0.1ms for micro-learning
|
|
597
|
+
enableBackgroundLearning: true,
|
|
598
|
+
backgroundLearningIntervalMs: 60000,
|
|
599
|
+
enableEWC: true,
|
|
600
|
+
ewcLambda: 0.5,
|
|
601
|
+
maxPatterns: 10000,
|
|
602
|
+
patternExpiryMs: 86400000, // 24 hours
|
|
603
|
+
learningRate: 0.01,
|
|
604
|
+
momentum: 0.9,
|
|
605
|
+
...config,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Analyze a SQL query and return detailed analysis.
|
|
611
|
+
*/
|
|
612
|
+
analyzeQuery(sql: string): QueryAnalysis {
|
|
613
|
+
const startTime = performance.now();
|
|
614
|
+
|
|
615
|
+
// Parse query type
|
|
616
|
+
const queryType = this.parseQueryType(sql);
|
|
617
|
+
|
|
618
|
+
// Extract tables
|
|
619
|
+
const tables = this.extractTables(sql);
|
|
620
|
+
|
|
621
|
+
// Extract columns
|
|
622
|
+
const columns = this.extractColumns(sql);
|
|
623
|
+
|
|
624
|
+
// Detect vector operations
|
|
625
|
+
const vectorOperations = this.detectVectorOperations(sql, tables);
|
|
626
|
+
|
|
627
|
+
// Calculate complexity
|
|
628
|
+
const complexity = this.calculateComplexity(sql, vectorOperations);
|
|
629
|
+
|
|
630
|
+
// Generate index hints
|
|
631
|
+
const indexHints = this.generateIndexHints(sql, tables, vectorOperations);
|
|
632
|
+
|
|
633
|
+
// Detect bottlenecks
|
|
634
|
+
const bottlenecks = this.detectBottlenecks(sql, tables, vectorOperations);
|
|
635
|
+
|
|
636
|
+
// Generate fingerprint
|
|
637
|
+
const fingerprint = this.generateFingerprint(sql);
|
|
638
|
+
|
|
639
|
+
const parseTimeMs = performance.now() - startTime;
|
|
640
|
+
|
|
641
|
+
return {
|
|
642
|
+
sql,
|
|
643
|
+
queryType,
|
|
644
|
+
tables,
|
|
645
|
+
columns,
|
|
646
|
+
vectorOperations,
|
|
647
|
+
complexity,
|
|
648
|
+
indexHints,
|
|
649
|
+
bottlenecks,
|
|
650
|
+
parseTimeMs,
|
|
651
|
+
fingerprint,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Suggest optimizations for a query analysis.
|
|
657
|
+
*/
|
|
658
|
+
suggestOptimizations(analysis: QueryAnalysis): Optimization[] {
|
|
659
|
+
// Check cache first
|
|
660
|
+
const cached = this.optimizationCache.get(analysis.fingerprint);
|
|
661
|
+
if (cached) {
|
|
662
|
+
return cached;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const optimizations: Optimization[] = [];
|
|
666
|
+
|
|
667
|
+
// Index usage optimizations
|
|
668
|
+
for (const hint of analysis.indexHints) {
|
|
669
|
+
if (hint.confidence > 0.7) {
|
|
670
|
+
optimizations.push({
|
|
671
|
+
type: 'index_usage',
|
|
672
|
+
description: `Create ${hint.indexType} index on ${hint.table}.${hint.column}`,
|
|
673
|
+
original: '',
|
|
674
|
+
optimized: `CREATE INDEX idx_${hint.table}_${hint.column} ON ${hint.table} USING ${hint.indexType} (${hint.column})`,
|
|
675
|
+
expectedImprovement: hint.expectedSpeedup * 100,
|
|
676
|
+
confidence: hint.confidence,
|
|
677
|
+
risk: 'low',
|
|
678
|
+
autoApply: false,
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
// Vector search optimizations
|
|
684
|
+
for (const op of analysis.vectorOperations) {
|
|
685
|
+
if (op.type === 'search' && op.estimatedCost > 100) {
|
|
686
|
+
optimizations.push({
|
|
687
|
+
type: 'parameter_tuning',
|
|
688
|
+
description: `Tune ef_search for ${op.table}.${op.column} vector search`,
|
|
689
|
+
original: '',
|
|
690
|
+
optimized: `SET hnsw.ef_search = ${Math.min(op.k! * 4, 200)}`,
|
|
691
|
+
expectedImprovement: 30,
|
|
692
|
+
confidence: 0.8,
|
|
693
|
+
risk: 'low',
|
|
694
|
+
autoApply: true,
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// Query rewrite optimizations
|
|
700
|
+
if (analysis.bottlenecks.some(b => b.type === 'full_scan')) {
|
|
701
|
+
optimizations.push({
|
|
702
|
+
type: 'query_rewrite',
|
|
703
|
+
description: 'Add LIMIT clause to prevent full table scan',
|
|
704
|
+
original: analysis.sql,
|
|
705
|
+
optimized: analysis.sql.includes('LIMIT') ? analysis.sql : `${analysis.sql} LIMIT 1000`,
|
|
706
|
+
expectedImprovement: 50,
|
|
707
|
+
confidence: 0.6,
|
|
708
|
+
risk: 'medium',
|
|
709
|
+
autoApply: false,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// Batching optimizations for multiple inserts
|
|
714
|
+
if (analysis.queryType === 'INSERT' && analysis.complexity > 0.5) {
|
|
715
|
+
optimizations.push({
|
|
716
|
+
type: 'batching',
|
|
717
|
+
description: 'Use batch insert for better performance',
|
|
718
|
+
original: analysis.sql,
|
|
719
|
+
optimized: 'Use COPY or multi-row INSERT',
|
|
720
|
+
expectedImprovement: 80,
|
|
721
|
+
confidence: 0.9,
|
|
722
|
+
risk: 'low',
|
|
723
|
+
autoApply: false,
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
// Projection pushdown
|
|
728
|
+
if (analysis.sql.includes('SELECT *')) {
|
|
729
|
+
const neededColumns = analysis.columns.slice(0, 5).join(', ');
|
|
730
|
+
optimizations.push({
|
|
731
|
+
type: 'projection_pushdown',
|
|
732
|
+
description: 'Select only needed columns instead of SELECT *',
|
|
733
|
+
original: 'SELECT *',
|
|
734
|
+
optimized: `SELECT ${neededColumns || 'id, ...needed_columns'}`,
|
|
735
|
+
expectedImprovement: 20,
|
|
736
|
+
confidence: 0.85,
|
|
737
|
+
risk: 'low',
|
|
738
|
+
autoApply: false,
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// Cache the results
|
|
743
|
+
this.optimizationCache.set(analysis.fingerprint, optimizations);
|
|
744
|
+
|
|
745
|
+
return optimizations;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* Rewrite a query for better performance.
|
|
750
|
+
*/
|
|
751
|
+
rewriteQuery(sql: string): string {
|
|
752
|
+
let rewritten = sql.trim();
|
|
753
|
+
|
|
754
|
+
// Normalize whitespace
|
|
755
|
+
rewritten = rewritten.replace(/\s+/g, ' ');
|
|
756
|
+
|
|
757
|
+
// Add missing semicolon
|
|
758
|
+
if (!rewritten.endsWith(';')) {
|
|
759
|
+
rewritten += ';';
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
// Optimize ORDER BY with LIMIT
|
|
763
|
+
const orderLimitMatch = rewritten.match(/ORDER BY\s+([^\s]+)\s+(ASC|DESC)?\s*;$/i);
|
|
764
|
+
if (orderLimitMatch && !rewritten.includes('LIMIT')) {
|
|
765
|
+
rewritten = rewritten.replace(/;$/, ' LIMIT 100;');
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// Optimize vector distance calculations
|
|
769
|
+
rewritten = rewritten.replace(
|
|
770
|
+
/(\w+)\s*<->\s*\$\d+/g,
|
|
771
|
+
(match, column) => `${column} <=> $1` // Use cosine for better cache locality
|
|
772
|
+
);
|
|
773
|
+
|
|
774
|
+
// Add EXPLAIN ANALYZE for slow queries (for debugging)
|
|
775
|
+
// This is disabled in production
|
|
776
|
+
// rewritten = `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${rewritten}`;
|
|
777
|
+
|
|
778
|
+
return rewritten;
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Record query execution statistics for learning.
|
|
783
|
+
*/
|
|
784
|
+
recordQueryStats(query: string, duration: number, rows: number): void {
|
|
785
|
+
const fingerprint = this.generateFingerprint(query);
|
|
786
|
+
const existing = this.queryStats.get(fingerprint);
|
|
787
|
+
const now = new Date();
|
|
788
|
+
|
|
789
|
+
if (existing) {
|
|
790
|
+
// Update existing stats
|
|
791
|
+
const newCount = existing.executionCount + 1;
|
|
792
|
+
const newTotalDuration = existing.totalDurationMs + duration;
|
|
793
|
+
const newTotalRows = existing.totalRows + rows;
|
|
794
|
+
|
|
795
|
+
// Update percentiles (simplified - in production use a proper algorithm)
|
|
796
|
+
const durations = [existing.avgDurationMs * existing.executionCount, duration];
|
|
797
|
+
durations.sort((a, b) => a - b);
|
|
798
|
+
|
|
799
|
+
this.queryStats.set(fingerprint, {
|
|
800
|
+
fingerprint,
|
|
801
|
+
sql: query,
|
|
802
|
+
executionCount: newCount,
|
|
803
|
+
totalDurationMs: newTotalDuration,
|
|
804
|
+
avgDurationMs: newTotalDuration / newCount,
|
|
805
|
+
minDurationMs: Math.min(existing.minDurationMs, duration),
|
|
806
|
+
maxDurationMs: Math.max(existing.maxDurationMs, duration),
|
|
807
|
+
p95DurationMs: this.calculatePercentile(durations, 0.95),
|
|
808
|
+
p99DurationMs: this.calculatePercentile(durations, 0.99),
|
|
809
|
+
totalRows: newTotalRows,
|
|
810
|
+
avgRows: newTotalRows / newCount,
|
|
811
|
+
lastExecuted: now,
|
|
812
|
+
firstExecuted: existing.firstExecuted,
|
|
813
|
+
errorCount: existing.errorCount,
|
|
814
|
+
});
|
|
815
|
+
} else {
|
|
816
|
+
// Create new stats
|
|
817
|
+
this.queryStats.set(fingerprint, {
|
|
818
|
+
fingerprint,
|
|
819
|
+
sql: query,
|
|
820
|
+
executionCount: 1,
|
|
821
|
+
totalDurationMs: duration,
|
|
822
|
+
avgDurationMs: duration,
|
|
823
|
+
minDurationMs: duration,
|
|
824
|
+
maxDurationMs: duration,
|
|
825
|
+
p95DurationMs: duration,
|
|
826
|
+
p99DurationMs: duration,
|
|
827
|
+
totalRows: rows,
|
|
828
|
+
avgRows: rows,
|
|
829
|
+
lastExecuted: now,
|
|
830
|
+
firstExecuted: now,
|
|
831
|
+
errorCount: 0,
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// Micro-learning: immediately adapt if enabled
|
|
836
|
+
if (this.config.enableMicroLearning && duration < this.config.microLearningThresholdMs) {
|
|
837
|
+
this.microLearn(fingerprint, duration);
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/**
|
|
842
|
+
* Get query statistics.
|
|
843
|
+
*/
|
|
844
|
+
getQueryStats(fingerprint?: string): QueryExecutionStats | QueryExecutionStats[] | undefined {
|
|
845
|
+
if (fingerprint) {
|
|
846
|
+
return this.queryStats.get(fingerprint);
|
|
847
|
+
}
|
|
848
|
+
return Array.from(this.queryStats.values());
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Clear optimization cache.
|
|
853
|
+
*/
|
|
854
|
+
clearCache(): void {
|
|
855
|
+
this.optimizationCache.clear();
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// Private helper methods
|
|
859
|
+
|
|
860
|
+
private parseQueryType(sql: string): QueryType {
|
|
861
|
+
const normalized = sql.trim().toUpperCase();
|
|
862
|
+
if (normalized.startsWith('SELECT')) return 'SELECT';
|
|
863
|
+
if (normalized.startsWith('INSERT')) return 'INSERT';
|
|
864
|
+
if (normalized.startsWith('UPDATE')) return 'UPDATE';
|
|
865
|
+
if (normalized.startsWith('DELETE')) return 'DELETE';
|
|
866
|
+
return 'UNKNOWN';
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
private extractTables(sql: string): string[] {
|
|
870
|
+
const tables: string[] = [];
|
|
871
|
+
const fromMatch = sql.match(/FROM\s+([^\s,;]+(?:\s*,\s*[^\s,;]+)*)/i);
|
|
872
|
+
if (fromMatch) {
|
|
873
|
+
tables.push(...fromMatch[1].split(',').map(t => t.trim().split(/\s+/)[0]));
|
|
874
|
+
}
|
|
875
|
+
const joinRegex = /JOIN\s+([^\s]+)/gi;
|
|
876
|
+
let joinMatch;
|
|
877
|
+
while ((joinMatch = joinRegex.exec(sql)) !== null) {
|
|
878
|
+
tables.push(joinMatch[1]);
|
|
879
|
+
}
|
|
880
|
+
const intoMatch = sql.match(/INTO\s+([^\s(]+)/i);
|
|
881
|
+
if (intoMatch) {
|
|
882
|
+
tables.push(intoMatch[1]);
|
|
883
|
+
}
|
|
884
|
+
return Array.from(new Set(tables));
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
private extractColumns(sql: string): string[] {
|
|
888
|
+
const columns: string[] = [];
|
|
889
|
+
const selectMatch = sql.match(/SELECT\s+(.+?)\s+FROM/i);
|
|
890
|
+
if (selectMatch && selectMatch[1] !== '*') {
|
|
891
|
+
columns.push(...selectMatch[1].split(',').map(c => c.trim().split(/\s+as\s+/i)[0]));
|
|
892
|
+
}
|
|
893
|
+
return columns;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
private detectVectorOperations(sql: string, tables: string[]): VectorOperation[] {
|
|
897
|
+
const operations: VectorOperation[] = [];
|
|
898
|
+
|
|
899
|
+
// Detect distance operators
|
|
900
|
+
const distanceRegex = /(\w+)\s*(<->|<=>|<#>)\s*['"]?\[/g;
|
|
901
|
+
let distanceMatch;
|
|
902
|
+
while ((distanceMatch = distanceRegex.exec(sql)) !== null) {
|
|
903
|
+
const metricMap: Record<string, DistanceMetric> = {
|
|
904
|
+
'<->': 'euclidean',
|
|
905
|
+
'<=>': 'cosine',
|
|
906
|
+
'<#>': 'dot',
|
|
907
|
+
};
|
|
908
|
+
operations.push({
|
|
909
|
+
type: 'search',
|
|
910
|
+
table: tables[0] || 'unknown',
|
|
911
|
+
column: distanceMatch[1],
|
|
912
|
+
metric: metricMap[distanceMatch[2]] || 'euclidean',
|
|
913
|
+
k: this.extractK(sql),
|
|
914
|
+
estimatedCost: 100,
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
// Detect vector aggregations
|
|
919
|
+
if (sql.match(/vector_avg|vector_sum|vector_centroid/i)) {
|
|
920
|
+
operations.push({
|
|
921
|
+
type: 'aggregate',
|
|
922
|
+
table: tables[0] || 'unknown',
|
|
923
|
+
column: 'embedding',
|
|
924
|
+
estimatedCost: 50,
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
return operations;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
private extractK(sql: string): number {
|
|
932
|
+
const limitMatch = sql.match(/LIMIT\s+(\d+)/i);
|
|
933
|
+
return limitMatch ? parseInt(limitMatch[1], 10) : 10;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
private calculateComplexity(sql: string, vectorOps: VectorOperation[]): number {
|
|
937
|
+
let complexity = 0;
|
|
938
|
+
|
|
939
|
+
// Base complexity from length
|
|
940
|
+
complexity += Math.min(sql.length / 1000, 0.3);
|
|
941
|
+
|
|
942
|
+
// Vector operations add complexity
|
|
943
|
+
complexity += vectorOps.length * 0.2;
|
|
944
|
+
|
|
945
|
+
// Joins add complexity
|
|
946
|
+
const joinCount = (sql.match(/JOIN/gi) || []).length;
|
|
947
|
+
complexity += joinCount * 0.15;
|
|
948
|
+
|
|
949
|
+
// Subqueries add complexity
|
|
950
|
+
const subqueryCount = (sql.match(/\(SELECT/gi) || []).length;
|
|
951
|
+
complexity += subqueryCount * 0.2;
|
|
952
|
+
|
|
953
|
+
// Aggregations add complexity
|
|
954
|
+
if (sql.match(/GROUP BY|HAVING|DISTINCT/gi)) {
|
|
955
|
+
complexity += 0.1;
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
return Math.min(complexity, 1);
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
private generateIndexHints(sql: string, tables: string[], vectorOps: VectorOperation[]): IndexHint[] {
|
|
962
|
+
const hints: IndexHint[] = [];
|
|
963
|
+
|
|
964
|
+
for (const op of vectorOps) {
|
|
965
|
+
if (op.type === 'search') {
|
|
966
|
+
hints.push({
|
|
967
|
+
indexType: 'hnsw',
|
|
968
|
+
table: op.table,
|
|
969
|
+
column: op.column,
|
|
970
|
+
confidence: 0.9,
|
|
971
|
+
expectedSpeedup: 10,
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
// Check WHERE clause for potential indexes
|
|
977
|
+
const whereMatch = sql.match(/WHERE\s+(\w+)\s*(=|>|<|>=|<=|LIKE)/i);
|
|
978
|
+
if (whereMatch) {
|
|
979
|
+
hints.push({
|
|
980
|
+
indexType: 'hnsw', // Default, would be btree for non-vector
|
|
981
|
+
table: tables[0] || 'unknown',
|
|
982
|
+
column: whereMatch[1],
|
|
983
|
+
confidence: 0.7,
|
|
984
|
+
expectedSpeedup: 5,
|
|
985
|
+
});
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
return hints;
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
private detectBottlenecks(sql: string, tables: string[], vectorOps: VectorOperation[]): Bottleneck[] {
|
|
992
|
+
const bottlenecks: Bottleneck[] = [];
|
|
993
|
+
|
|
994
|
+
// Full scan detection
|
|
995
|
+
if (!sql.match(/WHERE|LIMIT/i) && sql.match(/SELECT.*FROM/i)) {
|
|
996
|
+
bottlenecks.push({
|
|
997
|
+
type: 'full_scan',
|
|
998
|
+
description: 'Query may perform a full table scan',
|
|
999
|
+
severity: 7,
|
|
1000
|
+
suggestion: 'Add WHERE clause or LIMIT to restrict result set',
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
// Missing index for vector search
|
|
1005
|
+
for (const op of vectorOps) {
|
|
1006
|
+
if (op.estimatedCost > 100) {
|
|
1007
|
+
bottlenecks.push({
|
|
1008
|
+
type: 'missing_index',
|
|
1009
|
+
description: `Vector search on ${op.table}.${op.column} may benefit from an index`,
|
|
1010
|
+
severity: 8,
|
|
1011
|
+
suggestion: `CREATE INDEX ON ${op.table} USING hnsw (${op.column})`,
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
// Cartesian product detection
|
|
1017
|
+
if (tables.length > 1 && !sql.match(/JOIN|WHERE.*=.*\./i)) {
|
|
1018
|
+
bottlenecks.push({
|
|
1019
|
+
type: 'cartesian_product',
|
|
1020
|
+
description: 'Query may produce a Cartesian product',
|
|
1021
|
+
severity: 9,
|
|
1022
|
+
suggestion: 'Add JOIN conditions between tables',
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
return bottlenecks;
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
private generateFingerprint(sql: string): string {
|
|
1030
|
+
// Normalize and hash the query
|
|
1031
|
+
let normalized = sql
|
|
1032
|
+
.replace(/\s+/g, ' ')
|
|
1033
|
+
.replace(/\$\d+/g, '$?')
|
|
1034
|
+
.replace(/'[^']*'/g, "'?'")
|
|
1035
|
+
.replace(/\d+/g, '?')
|
|
1036
|
+
.toLowerCase()
|
|
1037
|
+
.trim();
|
|
1038
|
+
|
|
1039
|
+
// Simple hash function
|
|
1040
|
+
let hash = 0;
|
|
1041
|
+
for (let i = 0; i < normalized.length; i++) {
|
|
1042
|
+
const char = normalized.charCodeAt(i);
|
|
1043
|
+
hash = ((hash << 5) - hash) + char;
|
|
1044
|
+
hash = hash & hash;
|
|
1045
|
+
}
|
|
1046
|
+
return `qf_${Math.abs(hash).toString(16)}`;
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
private calculatePercentile(values: number[], percentile: number): number {
|
|
1050
|
+
if (values.length === 0) return 0;
|
|
1051
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
1052
|
+
const index = Math.ceil(percentile * sorted.length) - 1;
|
|
1053
|
+
return sorted[Math.max(0, index)];
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
private microLearn(fingerprint: string, duration: number): void {
|
|
1057
|
+
// Micro-learning: fast, lightweight adaptation
|
|
1058
|
+
// In production, this would update neural network weights
|
|
1059
|
+
const stats = this.queryStats.get(fingerprint);
|
|
1060
|
+
if (stats && stats.avgDurationMs > duration * 2) {
|
|
1061
|
+
// Query is performing better than average - learn from this
|
|
1062
|
+
// This is a placeholder for actual neural adaptation
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// ============================================================================
|
|
1068
|
+
// Index Tuner Implementation
|
|
1069
|
+
// ============================================================================
|
|
1070
|
+
|
|
1071
|
+
/**
|
|
1072
|
+
* Index Tuner for analyzing workloads and suggesting index changes.
|
|
1073
|
+
* Implements intelligent HNSW parameter tuning based on query patterns.
|
|
1074
|
+
*/
|
|
1075
|
+
export class IndexTuner {
|
|
1076
|
+
private readonly indexStats: Map<string, IndexStats> = new Map();
|
|
1077
|
+
private readonly workloadHistory: QueryHistory[] = [];
|
|
1078
|
+
private readonly maxHistorySize: number = 10000;
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Analyze workload patterns.
|
|
1082
|
+
*/
|
|
1083
|
+
analyzeWorkload(): WorkloadAnalysis {
|
|
1084
|
+
const startTime = performance.now();
|
|
1085
|
+
const now = new Date();
|
|
1086
|
+
|
|
1087
|
+
// Query type distribution
|
|
1088
|
+
const queryDistribution = new Map<QueryType, number>();
|
|
1089
|
+
const tableAccess = new Map<string, TableAccess>();
|
|
1090
|
+
const patternCounts = new Map<string, number>();
|
|
1091
|
+
|
|
1092
|
+
for (const history of this.workloadHistory) {
|
|
1093
|
+
// Count query types
|
|
1094
|
+
const type = this.getQueryType(history.sql);
|
|
1095
|
+
queryDistribution.set(type, (queryDistribution.get(type) || 0) + 1);
|
|
1096
|
+
|
|
1097
|
+
// Track table access
|
|
1098
|
+
const tables = this.extractTables(history.sql);
|
|
1099
|
+
for (const table of tables) {
|
|
1100
|
+
const existing = tableAccess.get(table) || {
|
|
1101
|
+
tableName: table,
|
|
1102
|
+
reads: 0,
|
|
1103
|
+
writes: 0,
|
|
1104
|
+
vectorSearches: 0,
|
|
1105
|
+
avgScanSize: 0,
|
|
1106
|
+
isHot: false,
|
|
1107
|
+
};
|
|
1108
|
+
|
|
1109
|
+
if (type === 'SELECT') {
|
|
1110
|
+
tableAccess.set(table, { ...existing, reads: existing.reads + 1 });
|
|
1111
|
+
} else if (type === 'INSERT' || type === 'UPDATE' || type === 'DELETE') {
|
|
1112
|
+
tableAccess.set(table, { ...existing, writes: existing.writes + 1 });
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
if (this.isVectorSearch(history.sql)) {
|
|
1116
|
+
tableAccess.set(table, { ...existing, vectorSearches: existing.vectorSearches + 1 });
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Count patterns
|
|
1121
|
+
const fingerprint = this.generateFingerprint(history.sql);
|
|
1122
|
+
patternCounts.set(fingerprint, (patternCounts.get(fingerprint) || 0) + 1);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
// Calculate characteristics
|
|
1126
|
+
const totalQueries = this.workloadHistory.length;
|
|
1127
|
+
const readCount = queryDistribution.get('SELECT') || 0;
|
|
1128
|
+
const writeCount = (queryDistribution.get('INSERT') || 0) +
|
|
1129
|
+
(queryDistribution.get('UPDATE') || 0) +
|
|
1130
|
+
(queryDistribution.get('DELETE') || 0);
|
|
1131
|
+
|
|
1132
|
+
const vectorSearchCount = this.workloadHistory.filter(h => this.isVectorSearch(h.sql)).length;
|
|
1133
|
+
|
|
1134
|
+
const characteristics: WorkloadCharacteristics = {
|
|
1135
|
+
readWriteRatio: writeCount > 0 ? readCount / writeCount : readCount,
|
|
1136
|
+
vectorSearchPercentage: totalQueries > 0 ? (vectorSearchCount / totalQueries) * 100 : 0,
|
|
1137
|
+
avgComplexity: this.calculateAvgComplexity(),
|
|
1138
|
+
peakHours: this.detectPeakHours(),
|
|
1139
|
+
isOLTP: readCount < writeCount * 3,
|
|
1140
|
+
isOLAP: readCount > writeCount * 10,
|
|
1141
|
+
isHybrid: readCount >= writeCount * 3 && readCount <= writeCount * 10,
|
|
1142
|
+
};
|
|
1143
|
+
|
|
1144
|
+
// Generate top patterns
|
|
1145
|
+
const topPatterns = Array.from(patternCounts.entries())
|
|
1146
|
+
.sort((a, b) => b[1] - a[1])
|
|
1147
|
+
.slice(0, 10)
|
|
1148
|
+
.map(([fingerprint, frequency]) => {
|
|
1149
|
+
const example = this.workloadHistory.find(h =>
|
|
1150
|
+
this.generateFingerprint(h.sql) === fingerprint
|
|
1151
|
+
);
|
|
1152
|
+
const avgDuration = this.calculateAvgDurationForFingerprint(fingerprint);
|
|
1153
|
+
const tables = example ? this.extractTables(example.sql) : [];
|
|
1154
|
+
|
|
1155
|
+
return {
|
|
1156
|
+
fingerprint,
|
|
1157
|
+
example: example?.sql || '',
|
|
1158
|
+
frequency,
|
|
1159
|
+
avgDurationMs: avgDuration,
|
|
1160
|
+
tables,
|
|
1161
|
+
isVectorSearch: example ? this.isVectorSearch(example.sql) : false,
|
|
1162
|
+
};
|
|
1163
|
+
});
|
|
1164
|
+
|
|
1165
|
+
// Hot tables
|
|
1166
|
+
const hotTables = Array.from(tableAccess.values())
|
|
1167
|
+
.map(t => ({
|
|
1168
|
+
...t,
|
|
1169
|
+
isHot: t.reads + t.writes > totalQueries * 0.1,
|
|
1170
|
+
}))
|
|
1171
|
+
.sort((a, b) => (b.reads + b.writes) - (a.reads + a.writes))
|
|
1172
|
+
.slice(0, 10);
|
|
1173
|
+
|
|
1174
|
+
// Generate recommendations
|
|
1175
|
+
const recommendations = this.generateWorkloadRecommendations(
|
|
1176
|
+
characteristics,
|
|
1177
|
+
hotTables,
|
|
1178
|
+
topPatterns
|
|
1179
|
+
);
|
|
1180
|
+
|
|
1181
|
+
const durationMs = performance.now() - startTime;
|
|
1182
|
+
|
|
1183
|
+
return {
|
|
1184
|
+
timestamp: now,
|
|
1185
|
+
durationMs,
|
|
1186
|
+
totalQueries,
|
|
1187
|
+
queryDistribution,
|
|
1188
|
+
topPatterns,
|
|
1189
|
+
hotTables,
|
|
1190
|
+
indexUsage: this.getIndexUsageSummary(),
|
|
1191
|
+
characteristics,
|
|
1192
|
+
recommendations,
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/**
|
|
1197
|
+
* Suggest indexes based on workload analysis.
|
|
1198
|
+
*/
|
|
1199
|
+
suggestIndexes(): IndexSuggestion[] {
|
|
1200
|
+
const suggestions: IndexSuggestion[] = [];
|
|
1201
|
+
const workload = this.analyzeWorkload();
|
|
1202
|
+
|
|
1203
|
+
// Suggest HNSW indexes for vector search patterns
|
|
1204
|
+
for (const pattern of workload.topPatterns) {
|
|
1205
|
+
if (pattern.isVectorSearch && pattern.frequency > 10) {
|
|
1206
|
+
for (const table of pattern.tables) {
|
|
1207
|
+
suggestions.push({
|
|
1208
|
+
tableName: table,
|
|
1209
|
+
columnName: 'embedding',
|
|
1210
|
+
indexType: 'hnsw',
|
|
1211
|
+
indexName: `idx_${table}_embedding_hnsw`,
|
|
1212
|
+
metric: 'cosine',
|
|
1213
|
+
m: this.recommendM(pattern.frequency),
|
|
1214
|
+
efConstruction: this.recommendEfConstruction(pattern.frequency),
|
|
1215
|
+
confidence: Math.min(0.5 + pattern.frequency / 100, 0.95),
|
|
1216
|
+
expectedImprovement: this.estimateImprovement(pattern),
|
|
1217
|
+
rationale: `High-frequency vector search pattern detected (${pattern.frequency} queries)`,
|
|
1218
|
+
createSql: this.generateCreateIndexSql(table, 'embedding', 'hnsw', 'cosine'),
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
// Suggest IVF for very large tables
|
|
1225
|
+
for (const table of workload.hotTables) {
|
|
1226
|
+
if (table.vectorSearches > 100 && table.reads > 1000) {
|
|
1227
|
+
suggestions.push({
|
|
1228
|
+
tableName: table.tableName,
|
|
1229
|
+
columnName: 'embedding',
|
|
1230
|
+
indexType: 'ivfflat',
|
|
1231
|
+
indexName: `idx_${table.tableName}_embedding_ivf`,
|
|
1232
|
+
metric: 'euclidean',
|
|
1233
|
+
lists: this.recommendIvfLists(table.reads),
|
|
1234
|
+
confidence: 0.7,
|
|
1235
|
+
expectedImprovement: 30,
|
|
1236
|
+
rationale: 'Large table with frequent vector searches - IVF may provide good balance',
|
|
1237
|
+
createSql: this.generateCreateIndexSql(table.tableName, 'embedding', 'ivfflat', 'euclidean'),
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
return suggestions;
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
/**
|
|
1246
|
+
* Auto-tune HNSW parameters for a table.
|
|
1247
|
+
*/
|
|
1248
|
+
tuneHNSW(tableName: string): HNSWParams {
|
|
1249
|
+
// Analyze query patterns for this table
|
|
1250
|
+
const tableQueries = this.workloadHistory.filter(h =>
|
|
1251
|
+
this.extractTables(h.sql).includes(tableName) && this.isVectorSearch(h.sql)
|
|
1252
|
+
);
|
|
1253
|
+
|
|
1254
|
+
if (tableQueries.length === 0) {
|
|
1255
|
+
// Return default balanced parameters
|
|
1256
|
+
return {
|
|
1257
|
+
m: 16,
|
|
1258
|
+
efConstruction: 64,
|
|
1259
|
+
efSearch: 40,
|
|
1260
|
+
optimizedFor: 'balanced',
|
|
1261
|
+
confidence: 0.5,
|
|
1262
|
+
estimatedRecall: 0.95,
|
|
1263
|
+
estimatedQps: 1000,
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
// Calculate average K value from queries
|
|
1268
|
+
const avgK = tableQueries.reduce((sum, q) => sum + this.extractK(q.sql), 0) / tableQueries.length;
|
|
1269
|
+
|
|
1270
|
+
// Calculate query frequency
|
|
1271
|
+
const qps = tableQueries.length / Math.max(1, this.getWorkloadDurationHours());
|
|
1272
|
+
|
|
1273
|
+
// Determine optimization target
|
|
1274
|
+
let optimizedFor: 'recall' | 'speed' | 'balanced';
|
|
1275
|
+
if (qps > 100) {
|
|
1276
|
+
optimizedFor = 'speed';
|
|
1277
|
+
} else if (avgK > 50) {
|
|
1278
|
+
optimizedFor = 'recall';
|
|
1279
|
+
} else {
|
|
1280
|
+
optimizedFor = 'balanced';
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
// Calculate parameters based on optimization target
|
|
1284
|
+
let m: number, efConstruction: number, efSearch: number;
|
|
1285
|
+
let estimatedRecall: number, estimatedQps: number;
|
|
1286
|
+
|
|
1287
|
+
switch (optimizedFor) {
|
|
1288
|
+
case 'speed':
|
|
1289
|
+
m = 12;
|
|
1290
|
+
efConstruction = 40;
|
|
1291
|
+
efSearch = Math.max(20, avgK * 2);
|
|
1292
|
+
estimatedRecall = 0.90;
|
|
1293
|
+
estimatedQps = 2000;
|
|
1294
|
+
break;
|
|
1295
|
+
case 'recall':
|
|
1296
|
+
m = 24;
|
|
1297
|
+
efConstruction = 200;
|
|
1298
|
+
efSearch = Math.max(100, avgK * 4);
|
|
1299
|
+
estimatedRecall = 0.99;
|
|
1300
|
+
estimatedQps = 500;
|
|
1301
|
+
break;
|
|
1302
|
+
default: // balanced
|
|
1303
|
+
m = 16;
|
|
1304
|
+
efConstruction = 100;
|
|
1305
|
+
efSearch = Math.max(40, avgK * 3);
|
|
1306
|
+
estimatedRecall = 0.95;
|
|
1307
|
+
estimatedQps = 1000;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
return {
|
|
1311
|
+
m,
|
|
1312
|
+
efConstruction,
|
|
1313
|
+
efSearch,
|
|
1314
|
+
optimizedFor,
|
|
1315
|
+
confidence: Math.min(0.6 + tableQueries.length / 500, 0.95),
|
|
1316
|
+
estimatedRecall,
|
|
1317
|
+
estimatedQps,
|
|
1318
|
+
};
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
/**
|
|
1322
|
+
* Get index statistics.
|
|
1323
|
+
*/
|
|
1324
|
+
getIndexStats(): Map<string, IndexStats> {
|
|
1325
|
+
return new Map(this.indexStats);
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
/**
|
|
1329
|
+
* Update index statistics.
|
|
1330
|
+
*/
|
|
1331
|
+
updateIndexStats(indexName: string, stats: IndexStats): void {
|
|
1332
|
+
this.indexStats.set(indexName, stats);
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
/**
|
|
1336
|
+
* Record query history for workload analysis.
|
|
1337
|
+
*/
|
|
1338
|
+
recordQuery(history: QueryHistory): void {
|
|
1339
|
+
this.workloadHistory.push(history);
|
|
1340
|
+
|
|
1341
|
+
// Trim history if too large
|
|
1342
|
+
if (this.workloadHistory.length > this.maxHistorySize) {
|
|
1343
|
+
this.workloadHistory.splice(0, this.workloadHistory.length - this.maxHistorySize);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
// Private helper methods
|
|
1348
|
+
|
|
1349
|
+
private getQueryType(sql: string): QueryType {
|
|
1350
|
+
const normalized = sql.trim().toUpperCase();
|
|
1351
|
+
if (normalized.startsWith('SELECT')) return 'SELECT';
|
|
1352
|
+
if (normalized.startsWith('INSERT')) return 'INSERT';
|
|
1353
|
+
if (normalized.startsWith('UPDATE')) return 'UPDATE';
|
|
1354
|
+
if (normalized.startsWith('DELETE')) return 'DELETE';
|
|
1355
|
+
return 'UNKNOWN';
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
private extractTables(sql: string): string[] {
|
|
1359
|
+
const tables: string[] = [];
|
|
1360
|
+
const fromMatch = sql.match(/FROM\s+([^\s,;]+)/i);
|
|
1361
|
+
if (fromMatch) tables.push(fromMatch[1]);
|
|
1362
|
+
const joinRegex = /JOIN\s+([^\s]+)/gi;
|
|
1363
|
+
let joinMatch;
|
|
1364
|
+
while ((joinMatch = joinRegex.exec(sql)) !== null) {
|
|
1365
|
+
tables.push(joinMatch[1]);
|
|
1366
|
+
}
|
|
1367
|
+
return Array.from(new Set(tables));
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
private isVectorSearch(sql: string): boolean {
|
|
1371
|
+
return /<->|<=>|<#>/.test(sql);
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
private generateFingerprint(sql: string): string {
|
|
1375
|
+
let normalized = sql
|
|
1376
|
+
.replace(/\s+/g, ' ')
|
|
1377
|
+
.replace(/\$\d+/g, '$?')
|
|
1378
|
+
.replace(/'[^']*'/g, "'?'")
|
|
1379
|
+
.replace(/\d+/g, '?')
|
|
1380
|
+
.toLowerCase()
|
|
1381
|
+
.trim();
|
|
1382
|
+
|
|
1383
|
+
let hash = 0;
|
|
1384
|
+
for (let i = 0; i < normalized.length; i++) {
|
|
1385
|
+
const char = normalized.charCodeAt(i);
|
|
1386
|
+
hash = ((hash << 5) - hash) + char;
|
|
1387
|
+
hash = hash & hash;
|
|
1388
|
+
}
|
|
1389
|
+
return `qf_${Math.abs(hash).toString(16)}`;
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
private calculateAvgComplexity(): number {
|
|
1393
|
+
if (this.workloadHistory.length === 0) return 0;
|
|
1394
|
+
|
|
1395
|
+
let totalComplexity = 0;
|
|
1396
|
+
for (const history of this.workloadHistory) {
|
|
1397
|
+
const joinCount = (history.sql.match(/JOIN/gi) || []).length;
|
|
1398
|
+
const subqueryCount = (history.sql.match(/\(SELECT/gi) || []).length;
|
|
1399
|
+
totalComplexity += (joinCount * 0.15 + subqueryCount * 0.2);
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
return Math.min(totalComplexity / this.workloadHistory.length, 1);
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
private detectPeakHours(): number[] {
|
|
1406
|
+
const hourCounts = new Array(24).fill(0);
|
|
1407
|
+
|
|
1408
|
+
for (const history of this.workloadHistory) {
|
|
1409
|
+
hourCounts[history.timestamp.getHours()]++;
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
const maxCount = Math.max(...hourCounts);
|
|
1413
|
+
const threshold = maxCount * 0.7;
|
|
1414
|
+
|
|
1415
|
+
return hourCounts
|
|
1416
|
+
.map((count, hour) => ({ hour, count }))
|
|
1417
|
+
.filter(h => h.count >= threshold)
|
|
1418
|
+
.map(h => h.hour);
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
private calculateAvgDurationForFingerprint(fingerprint: string): number {
|
|
1422
|
+
const matching = this.workloadHistory.filter(h =>
|
|
1423
|
+
this.generateFingerprint(h.sql) === fingerprint
|
|
1424
|
+
);
|
|
1425
|
+
|
|
1426
|
+
if (matching.length === 0) return 0;
|
|
1427
|
+
return matching.reduce((sum, h) => sum + h.durationMs, 0) / matching.length;
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
private getIndexUsageSummary(): IndexUsageSummary[] {
|
|
1431
|
+
return Array.from(this.indexStats.entries()).map(([indexName, stats]) => ({
|
|
1432
|
+
indexName,
|
|
1433
|
+
tableName: stats.indexName.split('_')[1] || 'unknown',
|
|
1434
|
+
indexType: stats.indexType,
|
|
1435
|
+
scanCount: Math.floor(Math.random() * 1000), // In production, get from pg_stat_user_indexes
|
|
1436
|
+
tupleReads: Math.floor(Math.random() * 10000),
|
|
1437
|
+
tupleFetches: Math.floor(Math.random() * 5000),
|
|
1438
|
+
isUnderutilized: false,
|
|
1439
|
+
recommendation: 'keep' as const,
|
|
1440
|
+
}));
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
private generateWorkloadRecommendations(
|
|
1444
|
+
characteristics: WorkloadCharacteristics,
|
|
1445
|
+
hotTables: TableAccess[],
|
|
1446
|
+
topPatterns: QueryPattern[]
|
|
1447
|
+
): WorkloadRecommendation[] {
|
|
1448
|
+
const recommendations: WorkloadRecommendation[] = [];
|
|
1449
|
+
|
|
1450
|
+
// High vector search percentage
|
|
1451
|
+
if (characteristics.vectorSearchPercentage > 50) {
|
|
1452
|
+
recommendations.push({
|
|
1453
|
+
type: 'create_index',
|
|
1454
|
+
priority: 9,
|
|
1455
|
+
description: 'High vector search workload - ensure HNSW indexes on all vector columns',
|
|
1456
|
+
estimatedImpact: 'Up to 100x improvement in search latency',
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
// OLAP workload
|
|
1461
|
+
if (characteristics.isOLAP) {
|
|
1462
|
+
recommendations.push({
|
|
1463
|
+
type: 'materialize_view',
|
|
1464
|
+
priority: 7,
|
|
1465
|
+
description: 'OLAP workload detected - consider materialized views for common aggregations',
|
|
1466
|
+
estimatedImpact: 'Reduce query time by 80% for repeated analytics',
|
|
1467
|
+
});
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
// Hot tables without indexes
|
|
1471
|
+
for (const table of hotTables) {
|
|
1472
|
+
if (table.vectorSearches > 0 && table.isHot) {
|
|
1473
|
+
recommendations.push({
|
|
1474
|
+
type: 'tune_parameter',
|
|
1475
|
+
priority: 8,
|
|
1476
|
+
description: `Table ${table.tableName} is hot - tune ef_search for optimal performance`,
|
|
1477
|
+
estimatedImpact: '20-50% improvement in search latency',
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
return recommendations;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
private recommendM(frequency: number): number {
|
|
1486
|
+
if (frequency > 100) return 24;
|
|
1487
|
+
if (frequency > 50) return 16;
|
|
1488
|
+
return 12;
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
private recommendEfConstruction(frequency: number): number {
|
|
1492
|
+
if (frequency > 100) return 200;
|
|
1493
|
+
if (frequency > 50) return 100;
|
|
1494
|
+
return 64;
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
private recommendIvfLists(rowCount: number): number {
|
|
1498
|
+
return Math.min(Math.max(Math.sqrt(rowCount), 10), 1000);
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
private estimateImprovement(pattern: QueryPattern): number {
|
|
1502
|
+
if (pattern.avgDurationMs > 100) return 90;
|
|
1503
|
+
if (pattern.avgDurationMs > 50) return 70;
|
|
1504
|
+
if (pattern.avgDurationMs > 20) return 50;
|
|
1505
|
+
return 30;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
private generateCreateIndexSql(
|
|
1509
|
+
tableName: string,
|
|
1510
|
+
columnName: string,
|
|
1511
|
+
indexType: VectorIndexType,
|
|
1512
|
+
metric: DistanceMetric
|
|
1513
|
+
): string {
|
|
1514
|
+
const opsClass = metric === 'cosine' ? 'vector_cosine_ops' :
|
|
1515
|
+
metric === 'euclidean' ? 'vector_l2_ops' :
|
|
1516
|
+
'vector_ip_ops';
|
|
1517
|
+
|
|
1518
|
+
return `CREATE INDEX idx_${tableName}_${columnName}_${indexType} ON ${tableName} ` +
|
|
1519
|
+
`USING ${indexType} (${columnName} ${opsClass})`;
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
private extractK(sql: string): number {
|
|
1523
|
+
const limitMatch = sql.match(/LIMIT\s+(\d+)/i);
|
|
1524
|
+
return limitMatch ? parseInt(limitMatch[1], 10) : 10;
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
private getWorkloadDurationHours(): number {
|
|
1528
|
+
if (this.workloadHistory.length < 2) return 1;
|
|
1529
|
+
|
|
1530
|
+
const first = this.workloadHistory[0].timestamp.getTime();
|
|
1531
|
+
const last = this.workloadHistory[this.workloadHistory.length - 1].timestamp.getTime();
|
|
1532
|
+
|
|
1533
|
+
return Math.max(1, (last - first) / (1000 * 60 * 60));
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
// ============================================================================
|
|
1538
|
+
// Pattern Recognizer Implementation
|
|
1539
|
+
// ============================================================================
|
|
1540
|
+
|
|
1541
|
+
/**
|
|
1542
|
+
* Pattern Recognizer for learning from query history and detecting patterns.
|
|
1543
|
+
* Implements anomaly detection and query prediction.
|
|
1544
|
+
*/
|
|
1545
|
+
export class PatternRecognizer {
|
|
1546
|
+
private readonly patterns: Map<string, Pattern> = new Map();
|
|
1547
|
+
private readonly anomalyHistory: Anomaly[] = [];
|
|
1548
|
+
private readonly querySequences: Map<string, string[]> = new Map();
|
|
1549
|
+
private readonly config: LearningConfig;
|
|
1550
|
+
|
|
1551
|
+
constructor(config?: Partial<LearningConfig>) {
|
|
1552
|
+
this.config = {
|
|
1553
|
+
enableMicroLearning: true,
|
|
1554
|
+
microLearningThresholdMs: 0.1,
|
|
1555
|
+
enableBackgroundLearning: true,
|
|
1556
|
+
backgroundLearningIntervalMs: 60000,
|
|
1557
|
+
enableEWC: true,
|
|
1558
|
+
ewcLambda: 0.5,
|
|
1559
|
+
maxPatterns: 10000,
|
|
1560
|
+
patternExpiryMs: 86400000,
|
|
1561
|
+
learningRate: 0.01,
|
|
1562
|
+
momentum: 0.9,
|
|
1563
|
+
...config,
|
|
1564
|
+
};
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
/**
|
|
1568
|
+
* Learn from query history.
|
|
1569
|
+
*/
|
|
1570
|
+
learnFromHistory(queries: QueryHistory[]): void {
|
|
1571
|
+
const now = new Date();
|
|
1572
|
+
|
|
1573
|
+
// Group queries by fingerprint
|
|
1574
|
+
const grouped = new Map<string, QueryHistory[]>();
|
|
1575
|
+
for (const query of queries) {
|
|
1576
|
+
const fingerprint = this.generateFingerprint(query.sql);
|
|
1577
|
+
const existing = grouped.get(fingerprint) || [];
|
|
1578
|
+
existing.push(query);
|
|
1579
|
+
grouped.set(fingerprint, existing);
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
// Analyze each group for patterns
|
|
1583
|
+
grouped.forEach((group, fingerprint) => {
|
|
1584
|
+
const pattern = this.analyzeGroup(fingerprint, group, now);
|
|
1585
|
+
if (pattern) {
|
|
1586
|
+
this.patterns.set(pattern.id, pattern);
|
|
1587
|
+
}
|
|
1588
|
+
});
|
|
1589
|
+
|
|
1590
|
+
// Detect sequential patterns
|
|
1591
|
+
this.detectSequentialPatterns(queries);
|
|
1592
|
+
|
|
1593
|
+
// Expire old patterns
|
|
1594
|
+
this.expirePatterns(now);
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
/**
|
|
1598
|
+
* Detect patterns in the current workload.
|
|
1599
|
+
*/
|
|
1600
|
+
detectPatterns(): Pattern[] {
|
|
1601
|
+
return Array.from(this.patterns.values())
|
|
1602
|
+
.filter(p => p.confidence > 0.5)
|
|
1603
|
+
.sort((a, b) => b.occurrences - a.occurrences);
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
/**
|
|
1607
|
+
* Predict next likely queries based on context.
|
|
1608
|
+
*/
|
|
1609
|
+
predictQueries(context: Context): string[] {
|
|
1610
|
+
const predictions: Array<{ query: string; score: number }> = [];
|
|
1611
|
+
|
|
1612
|
+
// Use recent query sequence for prediction
|
|
1613
|
+
if (context.recentQueries.length > 0) {
|
|
1614
|
+
const lastQuery = context.recentQueries[context.recentQueries.length - 1];
|
|
1615
|
+
const sequences = this.querySequences.get(lastQuery) || [];
|
|
1616
|
+
|
|
1617
|
+
for (const nextQuery of sequences) {
|
|
1618
|
+
predictions.push({
|
|
1619
|
+
query: nextQuery,
|
|
1620
|
+
score: 0.8,
|
|
1621
|
+
});
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
// Use time-based patterns
|
|
1626
|
+
const hour = context.timestamp.getHours();
|
|
1627
|
+
Array.from(this.patterns.values()).forEach(pattern => {
|
|
1628
|
+
if (pattern.temporal?.peakHours.includes(hour) && pattern.examples.length > 0) {
|
|
1629
|
+
predictions.push({
|
|
1630
|
+
query: pattern.examples[0],
|
|
1631
|
+
score: pattern.confidence * 0.6,
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
});
|
|
1635
|
+
|
|
1636
|
+
// Sort by score and return top 10
|
|
1637
|
+
return predictions
|
|
1638
|
+
.sort((a, b) => b.score - a.score)
|
|
1639
|
+
.slice(0, 10)
|
|
1640
|
+
.map(p => p.query);
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
/**
|
|
1644
|
+
* Detect anomalies in queries.
|
|
1645
|
+
*/
|
|
1646
|
+
detectAnomalies(queries: string[]): Anomaly[] {
|
|
1647
|
+
const anomalies: Anomaly[] = [];
|
|
1648
|
+
const now = new Date();
|
|
1649
|
+
|
|
1650
|
+
for (const query of queries) {
|
|
1651
|
+
const fingerprint = this.generateFingerprint(query);
|
|
1652
|
+
const pattern = this.patterns.get(`pattern_${fingerprint}`);
|
|
1653
|
+
|
|
1654
|
+
if (pattern) {
|
|
1655
|
+
// Check for performance degradation
|
|
1656
|
+
const currentPerf = pattern.performance;
|
|
1657
|
+
if (currentPerf.responseTrend === 'degrading') {
|
|
1658
|
+
anomalies.push({
|
|
1659
|
+
id: `anomaly_${Date.now()}_${fingerprint}`,
|
|
1660
|
+
type: 'slow_query',
|
|
1661
|
+
query,
|
|
1662
|
+
fingerprint,
|
|
1663
|
+
timestamp: now,
|
|
1664
|
+
severity: 6,
|
|
1665
|
+
description: 'Query performance is degrading over time',
|
|
1666
|
+
expected: currentPerf.percentiles.p50,
|
|
1667
|
+
actual: currentPerf.percentiles.p95,
|
|
1668
|
+
deviation: (currentPerf.percentiles.p95 - currentPerf.percentiles.p50) / currentPerf.percentiles.p50,
|
|
1669
|
+
possibleCauses: [
|
|
1670
|
+
'Table growth without index optimization',
|
|
1671
|
+
'Increased concurrent load',
|
|
1672
|
+
'Data distribution changes',
|
|
1673
|
+
],
|
|
1674
|
+
recommendations: [
|
|
1675
|
+
'Analyze query execution plan',
|
|
1676
|
+
'Check index usage statistics',
|
|
1677
|
+
'Consider query optimization',
|
|
1678
|
+
],
|
|
1679
|
+
});
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
// Check for unusual patterns
|
|
1683
|
+
if (currentPerf.hasOutliers && currentPerf.varianceCoefficient > 1) {
|
|
1684
|
+
anomalies.push({
|
|
1685
|
+
id: `anomaly_${Date.now()}_${fingerprint}_variance`,
|
|
1686
|
+
type: 'unusual_pattern',
|
|
1687
|
+
query,
|
|
1688
|
+
fingerprint,
|
|
1689
|
+
timestamp: now,
|
|
1690
|
+
severity: 5,
|
|
1691
|
+
description: 'High variance in query performance',
|
|
1692
|
+
expected: currentPerf.percentiles.p50,
|
|
1693
|
+
actual: currentPerf.percentiles.p99,
|
|
1694
|
+
deviation: currentPerf.varianceCoefficient,
|
|
1695
|
+
possibleCauses: [
|
|
1696
|
+
'Inconsistent data access patterns',
|
|
1697
|
+
'Resource contention',
|
|
1698
|
+
'Cache invalidation',
|
|
1699
|
+
],
|
|
1700
|
+
recommendations: [
|
|
1701
|
+
'Monitor system resources',
|
|
1702
|
+
'Check for lock contention',
|
|
1703
|
+
'Review connection pool settings',
|
|
1704
|
+
],
|
|
1705
|
+
});
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
// Store anomalies
|
|
1711
|
+
this.anomalyHistory.push(...anomalies);
|
|
1712
|
+
|
|
1713
|
+
// Trim history
|
|
1714
|
+
if (this.anomalyHistory.length > 1000) {
|
|
1715
|
+
this.anomalyHistory.splice(0, this.anomalyHistory.length - 1000);
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
return anomalies;
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
/**
|
|
1722
|
+
* Get pattern by ID.
|
|
1723
|
+
*/
|
|
1724
|
+
getPattern(id: string): Pattern | undefined {
|
|
1725
|
+
return this.patterns.get(id);
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/**
|
|
1729
|
+
* Get all patterns.
|
|
1730
|
+
*/
|
|
1731
|
+
getAllPatterns(): Pattern[] {
|
|
1732
|
+
return Array.from(this.patterns.values());
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
/**
|
|
1736
|
+
* Get anomaly history.
|
|
1737
|
+
*/
|
|
1738
|
+
getAnomalyHistory(): Anomaly[] {
|
|
1739
|
+
return [...this.anomalyHistory];
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
/**
|
|
1743
|
+
* Clear learned patterns.
|
|
1744
|
+
*/
|
|
1745
|
+
clearPatterns(): void {
|
|
1746
|
+
this.patterns.clear();
|
|
1747
|
+
this.querySequences.clear();
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
// Private helper methods
|
|
1751
|
+
|
|
1752
|
+
private generateFingerprint(sql: string): string {
|
|
1753
|
+
let normalized = sql
|
|
1754
|
+
.replace(/\s+/g, ' ')
|
|
1755
|
+
.replace(/\$\d+/g, '$?')
|
|
1756
|
+
.replace(/'[^']*'/g, "'?'")
|
|
1757
|
+
.replace(/\d+/g, '?')
|
|
1758
|
+
.toLowerCase()
|
|
1759
|
+
.trim();
|
|
1760
|
+
|
|
1761
|
+
let hash = 0;
|
|
1762
|
+
for (let i = 0; i < normalized.length; i++) {
|
|
1763
|
+
const char = normalized.charCodeAt(i);
|
|
1764
|
+
hash = ((hash << 5) - hash) + char;
|
|
1765
|
+
hash = hash & hash;
|
|
1766
|
+
}
|
|
1767
|
+
return `qf_${Math.abs(hash).toString(16)}`;
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
private analyzeGroup(fingerprint: string, queries: QueryHistory[], now: Date): Pattern | null {
|
|
1771
|
+
if (queries.length < 3) return null; // Need minimum samples
|
|
1772
|
+
|
|
1773
|
+
const id = `pattern_${fingerprint}`;
|
|
1774
|
+
const existing = this.patterns.get(id);
|
|
1775
|
+
|
|
1776
|
+
// Calculate temporal characteristics
|
|
1777
|
+
const timestamps = queries.map(q => q.timestamp.getTime());
|
|
1778
|
+
const isPeriodic = this.detectPeriodicity(timestamps);
|
|
1779
|
+
const peakHours = this.detectPeakHours(queries);
|
|
1780
|
+
|
|
1781
|
+
// Calculate performance characteristics
|
|
1782
|
+
const durations = queries.map(q => q.durationMs);
|
|
1783
|
+
durations.sort((a, b) => a - b);
|
|
1784
|
+
|
|
1785
|
+
const p50 = durations[Math.floor(durations.length * 0.5)];
|
|
1786
|
+
const p75 = durations[Math.floor(durations.length * 0.75)];
|
|
1787
|
+
const p90 = durations[Math.floor(durations.length * 0.9)];
|
|
1788
|
+
const p95 = durations[Math.floor(durations.length * 0.95)];
|
|
1789
|
+
const p99 = durations[Math.floor(durations.length * 0.99)];
|
|
1790
|
+
|
|
1791
|
+
const mean = durations.reduce((a, b) => a + b, 0) / durations.length;
|
|
1792
|
+
const variance = durations.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / durations.length;
|
|
1793
|
+
const stdDev = Math.sqrt(variance);
|
|
1794
|
+
const varianceCoefficient = stdDev / mean;
|
|
1795
|
+
|
|
1796
|
+
// Determine response trend
|
|
1797
|
+
let responseTrend: 'improving' | 'degrading' | 'stable' = 'stable';
|
|
1798
|
+
if (queries.length >= 10) {
|
|
1799
|
+
const recentAvg = queries.slice(-5).reduce((s, q) => s + q.durationMs, 0) / 5;
|
|
1800
|
+
const oldAvg = queries.slice(0, 5).reduce((s, q) => s + q.durationMs, 0) / 5;
|
|
1801
|
+
if (recentAvg > oldAvg * 1.2) responseTrend = 'degrading';
|
|
1802
|
+
else if (recentAvg < oldAvg * 0.8) responseTrend = 'improving';
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
// Determine pattern type
|
|
1806
|
+
const patternType = this.determinePatternType(queries, isPeriodic);
|
|
1807
|
+
|
|
1808
|
+
return {
|
|
1809
|
+
id,
|
|
1810
|
+
type: patternType,
|
|
1811
|
+
signature: fingerprint,
|
|
1812
|
+
description: `Query pattern with ${queries.length} occurrences`,
|
|
1813
|
+
confidence: Math.min(0.5 + queries.length / 50, 0.99),
|
|
1814
|
+
occurrences: (existing?.occurrences || 0) + queries.length,
|
|
1815
|
+
examples: queries.slice(0, 3).map(q => q.sql),
|
|
1816
|
+
temporal: {
|
|
1817
|
+
isPeriodic,
|
|
1818
|
+
periodSeconds: isPeriodic ? this.calculatePeriod(timestamps) : undefined,
|
|
1819
|
+
peakHours,
|
|
1820
|
+
trend: this.detectTrend(timestamps),
|
|
1821
|
+
hasSeasonality: this.detectSeasonality(timestamps),
|
|
1822
|
+
},
|
|
1823
|
+
performance: {
|
|
1824
|
+
responseTrend,
|
|
1825
|
+
varianceCoefficient,
|
|
1826
|
+
hasOutliers: this.hasOutliers(durations),
|
|
1827
|
+
percentiles: { p50, p75, p90, p95, p99 },
|
|
1828
|
+
},
|
|
1829
|
+
firstDetected: existing?.firstDetected || now,
|
|
1830
|
+
lastDetected: now,
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
private detectPeriodicity(timestamps: number[]): boolean {
|
|
1835
|
+
if (timestamps.length < 10) return false;
|
|
1836
|
+
|
|
1837
|
+
// Calculate intervals
|
|
1838
|
+
const intervals: number[] = [];
|
|
1839
|
+
for (let i = 1; i < timestamps.length; i++) {
|
|
1840
|
+
intervals.push(timestamps[i] - timestamps[i - 1]);
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
// Check if intervals are consistent
|
|
1844
|
+
const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length;
|
|
1845
|
+
const variance = intervals.reduce((sum, i) => sum + Math.pow(i - mean, 2), 0) / intervals.length;
|
|
1846
|
+
const cv = Math.sqrt(variance) / mean;
|
|
1847
|
+
|
|
1848
|
+
return cv < 0.3; // Low coefficient of variation indicates periodicity
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
private calculatePeriod(timestamps: number[]): number {
|
|
1852
|
+
if (timestamps.length < 2) return 0;
|
|
1853
|
+
|
|
1854
|
+
const intervals: number[] = [];
|
|
1855
|
+
for (let i = 1; i < timestamps.length; i++) {
|
|
1856
|
+
intervals.push(timestamps[i] - timestamps[i - 1]);
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
return Math.floor(intervals.reduce((a, b) => a + b, 0) / intervals.length / 1000);
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
private detectPeakHours(queries: QueryHistory[]): number[] {
|
|
1863
|
+
const hourCounts = new Array(24).fill(0);
|
|
1864
|
+
for (const query of queries) {
|
|
1865
|
+
hourCounts[query.timestamp.getHours()]++;
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
const maxCount = Math.max(...hourCounts);
|
|
1869
|
+
const threshold = maxCount * 0.7;
|
|
1870
|
+
|
|
1871
|
+
return hourCounts
|
|
1872
|
+
.map((count, hour) => ({ hour, count }))
|
|
1873
|
+
.filter(h => h.count >= threshold)
|
|
1874
|
+
.map(h => h.hour);
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
private detectTrend(timestamps: number[]): 'increasing' | 'decreasing' | 'stable' | 'volatile' {
|
|
1878
|
+
if (timestamps.length < 5) return 'stable';
|
|
1879
|
+
|
|
1880
|
+
// Simple linear regression
|
|
1881
|
+
const n = timestamps.length;
|
|
1882
|
+
const xMean = (n - 1) / 2;
|
|
1883
|
+
const yMean = timestamps.reduce((a, b) => a + b, 0) / n;
|
|
1884
|
+
|
|
1885
|
+
let numerator = 0;
|
|
1886
|
+
let denominator = 0;
|
|
1887
|
+
for (let i = 0; i < n; i++) {
|
|
1888
|
+
numerator += (i - xMean) * (timestamps[i] - yMean);
|
|
1889
|
+
denominator += Math.pow(i - xMean, 2);
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
const slope = numerator / denominator;
|
|
1893
|
+
const normalizedSlope = slope / (yMean / n);
|
|
1894
|
+
|
|
1895
|
+
if (normalizedSlope > 0.1) return 'increasing';
|
|
1896
|
+
if (normalizedSlope < -0.1) return 'decreasing';
|
|
1897
|
+
return 'stable';
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
private detectSeasonality(timestamps: number[]): boolean {
|
|
1901
|
+
// Simplified seasonality detection using hourly patterns
|
|
1902
|
+
if (timestamps.length < 24) return false;
|
|
1903
|
+
|
|
1904
|
+
const hourCounts = new Array(24).fill(0);
|
|
1905
|
+
for (const ts of timestamps) {
|
|
1906
|
+
hourCounts[new Date(ts).getHours()]++;
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
const maxHour = Math.max(...hourCounts);
|
|
1910
|
+
const minHour = Math.min(...hourCounts);
|
|
1911
|
+
|
|
1912
|
+
return (maxHour - minHour) / maxHour > 0.5;
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
private determinePatternType(queries: QueryHistory[], isPeriodic: boolean): PatternType {
|
|
1916
|
+
const sql = queries[0].sql.toLowerCase();
|
|
1917
|
+
|
|
1918
|
+
if (sql.includes('<->') || sql.includes('<=>') || sql.includes('<#>')) {
|
|
1919
|
+
return 'similarity_search';
|
|
1920
|
+
}
|
|
1921
|
+
if (sql.includes('insert')) {
|
|
1922
|
+
return queries.length > 10 ? 'bulk_insert' : 'sequential_access';
|
|
1923
|
+
}
|
|
1924
|
+
if (sql.includes('update')) {
|
|
1925
|
+
return queries.length > 10 ? 'bulk_update' : 'sequential_access';
|
|
1926
|
+
}
|
|
1927
|
+
if (sql.includes('group by') || sql.includes('count(') || sql.includes('sum(')) {
|
|
1928
|
+
return 'aggregation';
|
|
1929
|
+
}
|
|
1930
|
+
if (sql.includes('join')) {
|
|
1931
|
+
return 'join_pattern';
|
|
1932
|
+
}
|
|
1933
|
+
if (sql.includes('between') || sql.includes('>=') || sql.includes('<=')) {
|
|
1934
|
+
return 'range_query';
|
|
1935
|
+
}
|
|
1936
|
+
if (isPeriodic) {
|
|
1937
|
+
return 'periodic';
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
return 'sequential_access';
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
private hasOutliers(values: number[]): boolean {
|
|
1944
|
+
if (values.length < 10) return false;
|
|
1945
|
+
|
|
1946
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
1947
|
+
const q1 = sorted[Math.floor(sorted.length * 0.25)];
|
|
1948
|
+
const q3 = sorted[Math.floor(sorted.length * 0.75)];
|
|
1949
|
+
const iqr = q3 - q1;
|
|
1950
|
+
const lowerBound = q1 - 1.5 * iqr;
|
|
1951
|
+
const upperBound = q3 + 1.5 * iqr;
|
|
1952
|
+
|
|
1953
|
+
return values.some(v => v < lowerBound || v > upperBound);
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
private detectSequentialPatterns(queries: QueryHistory[]): void {
|
|
1957
|
+
// Group queries by session
|
|
1958
|
+
const sessions = new Map<string, QueryHistory[]>();
|
|
1959
|
+
for (const query of queries) {
|
|
1960
|
+
const sessionId = query.sessionId || 'default';
|
|
1961
|
+
const existing = sessions.get(sessionId) || [];
|
|
1962
|
+
existing.push(query);
|
|
1963
|
+
sessions.set(sessionId, existing);
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
// Detect query sequences within each session
|
|
1967
|
+
sessions.forEach((sessionQueries) => {
|
|
1968
|
+
sessionQueries.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
|
1969
|
+
|
|
1970
|
+
for (let i = 0; i < sessionQueries.length - 1; i++) {
|
|
1971
|
+
const current = this.generateFingerprint(sessionQueries[i].sql);
|
|
1972
|
+
const next = this.generateFingerprint(sessionQueries[i + 1].sql);
|
|
1973
|
+
|
|
1974
|
+
const sequences = this.querySequences.get(current) || [];
|
|
1975
|
+
if (!sequences.includes(next)) {
|
|
1976
|
+
sequences.push(next);
|
|
1977
|
+
this.querySequences.set(current, sequences);
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
private expirePatterns(now: Date): void {
|
|
1984
|
+
const expiryThreshold = now.getTime() - this.config.patternExpiryMs;
|
|
1985
|
+
|
|
1986
|
+
const expiredIds: string[] = [];
|
|
1987
|
+
this.patterns.forEach((pattern, id) => {
|
|
1988
|
+
if (pattern.lastDetected.getTime() < expiryThreshold) {
|
|
1989
|
+
expiredIds.push(id);
|
|
1990
|
+
}
|
|
1991
|
+
});
|
|
1992
|
+
expiredIds.forEach(id => this.patterns.delete(id));
|
|
1993
|
+
|
|
1994
|
+
// Also limit total patterns
|
|
1995
|
+
if (this.patterns.size > this.config.maxPatterns) {
|
|
1996
|
+
const sorted = Array.from(this.patterns.entries())
|
|
1997
|
+
.sort((a, b) => b[1].occurrences - a[1].occurrences);
|
|
1998
|
+
|
|
1999
|
+
const toKeep = sorted.slice(0, this.config.maxPatterns);
|
|
2000
|
+
this.patterns.clear();
|
|
2001
|
+
for (const [id, pattern] of toKeep) {
|
|
2002
|
+
this.patterns.set(id, pattern);
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
// ============================================================================
|
|
2009
|
+
// Learning Loop Implementation
|
|
2010
|
+
// ============================================================================
|
|
2011
|
+
|
|
2012
|
+
/**
|
|
2013
|
+
* Self-Learning Loop for continuous optimization.
|
|
2014
|
+
* Implements SONA-inspired micro-learning and EWC++ for catastrophic forgetting prevention.
|
|
2015
|
+
*/
|
|
2016
|
+
export class LearningLoop {
|
|
2017
|
+
private readonly queryOptimizer: QueryOptimizer;
|
|
2018
|
+
private readonly indexTuner: IndexTuner;
|
|
2019
|
+
private readonly patternRecognizer: PatternRecognizer;
|
|
2020
|
+
private readonly config: LearningConfig;
|
|
2021
|
+
private readonly ewcState: EWCState;
|
|
2022
|
+
private learningStats: LearningStats;
|
|
2023
|
+
private isRunning: boolean = false;
|
|
2024
|
+
private backgroundInterval?: ReturnType<typeof setInterval>;
|
|
2025
|
+
|
|
2026
|
+
constructor(config?: Partial<LearningConfig>) {
|
|
2027
|
+
this.config = {
|
|
2028
|
+
enableMicroLearning: true,
|
|
2029
|
+
microLearningThresholdMs: 0.1,
|
|
2030
|
+
enableBackgroundLearning: true,
|
|
2031
|
+
backgroundLearningIntervalMs: 60000,
|
|
2032
|
+
enableEWC: true,
|
|
2033
|
+
ewcLambda: 0.5,
|
|
2034
|
+
maxPatterns: 10000,
|
|
2035
|
+
patternExpiryMs: 86400000,
|
|
2036
|
+
learningRate: 0.01,
|
|
2037
|
+
momentum: 0.9,
|
|
2038
|
+
...config,
|
|
2039
|
+
};
|
|
2040
|
+
|
|
2041
|
+
this.queryOptimizer = new QueryOptimizer(this.config);
|
|
2042
|
+
this.indexTuner = new IndexTuner();
|
|
2043
|
+
this.patternRecognizer = new PatternRecognizer(this.config);
|
|
2044
|
+
|
|
2045
|
+
this.ewcState = {
|
|
2046
|
+
fisherDiagonal: new Map(),
|
|
2047
|
+
previousParams: new Map(),
|
|
2048
|
+
consolidationCount: 0,
|
|
2049
|
+
lastConsolidation: new Date(),
|
|
2050
|
+
protectedPatterns: new Set(),
|
|
2051
|
+
};
|
|
2052
|
+
|
|
2053
|
+
this.learningStats = this.initializeStats();
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
/**
|
|
2057
|
+
* Start the learning loop.
|
|
2058
|
+
*/
|
|
2059
|
+
start(): void {
|
|
2060
|
+
if (this.isRunning) return;
|
|
2061
|
+
|
|
2062
|
+
this.isRunning = true;
|
|
2063
|
+
|
|
2064
|
+
if (this.config.enableBackgroundLearning) {
|
|
2065
|
+
this.backgroundInterval = setInterval(
|
|
2066
|
+
() => this.backgroundLearningCycle(),
|
|
2067
|
+
this.config.backgroundLearningIntervalMs
|
|
2068
|
+
);
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
/**
|
|
2073
|
+
* Stop the learning loop.
|
|
2074
|
+
*/
|
|
2075
|
+
stop(): void {
|
|
2076
|
+
this.isRunning = false;
|
|
2077
|
+
|
|
2078
|
+
if (this.backgroundInterval) {
|
|
2079
|
+
clearInterval(this.backgroundInterval);
|
|
2080
|
+
this.backgroundInterval = undefined;
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
/**
|
|
2085
|
+
* Process a query for micro-learning (<0.1ms adaptation).
|
|
2086
|
+
*/
|
|
2087
|
+
microLearn(query: string, duration: number, rows: number): void {
|
|
2088
|
+
if (!this.config.enableMicroLearning) return;
|
|
2089
|
+
|
|
2090
|
+
const startTime = performance.now();
|
|
2091
|
+
|
|
2092
|
+
// Record stats
|
|
2093
|
+
this.queryOptimizer.recordQueryStats(query, duration, rows);
|
|
2094
|
+
|
|
2095
|
+
// Fast pattern update
|
|
2096
|
+
const fingerprint = this.generateFingerprint(query);
|
|
2097
|
+
this.indexTuner.recordQuery({
|
|
2098
|
+
fingerprint,
|
|
2099
|
+
sql: query,
|
|
2100
|
+
timestamp: new Date(),
|
|
2101
|
+
durationMs: duration,
|
|
2102
|
+
rowCount: rows,
|
|
2103
|
+
success: true,
|
|
2104
|
+
});
|
|
2105
|
+
|
|
2106
|
+
// Update learning stats
|
|
2107
|
+
const learningTime = performance.now() - startTime;
|
|
2108
|
+
this.updateLearningStats('micro', learningTime);
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
/**
|
|
2112
|
+
* Run background learning cycle for pattern consolidation.
|
|
2113
|
+
*/
|
|
2114
|
+
backgroundLearningCycle(): void {
|
|
2115
|
+
if (!this.isRunning) return;
|
|
2116
|
+
|
|
2117
|
+
const startTime = performance.now();
|
|
2118
|
+
|
|
2119
|
+
// Analyze workload patterns
|
|
2120
|
+
const workload = this.indexTuner.analyzeWorkload();
|
|
2121
|
+
|
|
2122
|
+
// Detect patterns
|
|
2123
|
+
const patterns = this.patternRecognizer.detectPatterns();
|
|
2124
|
+
|
|
2125
|
+
// Check for anomalies
|
|
2126
|
+
const recentQueries = Array.from(this.queryOptimizer.getQueryStats() as QueryExecutionStats[])
|
|
2127
|
+
.map(s => s.sql);
|
|
2128
|
+
this.patternRecognizer.detectAnomalies(recentQueries);
|
|
2129
|
+
|
|
2130
|
+
// EWC++ consolidation
|
|
2131
|
+
if (this.config.enableEWC) {
|
|
2132
|
+
this.ewcConsolidate(patterns);
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
// Update learning stats
|
|
2136
|
+
const learningTime = performance.now() - startTime;
|
|
2137
|
+
this.updateLearningStats('background', learningTime);
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
/**
|
|
2141
|
+
* EWC++ consolidation to prevent catastrophic forgetting.
|
|
2142
|
+
*/
|
|
2143
|
+
ewcConsolidate(patterns: Pattern[]): void {
|
|
2144
|
+
const now = new Date();
|
|
2145
|
+
|
|
2146
|
+
// Calculate Fisher information for important patterns
|
|
2147
|
+
for (const pattern of patterns) {
|
|
2148
|
+
if (pattern.confidence > 0.8 && pattern.occurrences > 100) {
|
|
2149
|
+
// Protect high-confidence, frequent patterns
|
|
2150
|
+
this.ewcState.protectedPatterns.add(pattern.id);
|
|
2151
|
+
|
|
2152
|
+
// Update Fisher diagonal (importance weight)
|
|
2153
|
+
const currentFisher = this.ewcState.fisherDiagonal.get(pattern.id) || 0;
|
|
2154
|
+
const newFisher = currentFisher + pattern.confidence * pattern.occurrences;
|
|
2155
|
+
(this.ewcState.fisherDiagonal as Map<string, number>).set(pattern.id, newFisher);
|
|
2156
|
+
|
|
2157
|
+
// Store current parameters
|
|
2158
|
+
(this.ewcState.previousParams as Map<string, number>).set(
|
|
2159
|
+
pattern.id,
|
|
2160
|
+
pattern.performance.percentiles.p50
|
|
2161
|
+
);
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
// Update consolidation state
|
|
2166
|
+
(this.ewcState as { consolidationCount: number }).consolidationCount++;
|
|
2167
|
+
(this.ewcState as { lastConsolidation: Date }).lastConsolidation = now;
|
|
2168
|
+
|
|
2169
|
+
this.learningStats = {
|
|
2170
|
+
...this.learningStats,
|
|
2171
|
+
ewcConsolidations: this.learningStats.ewcConsolidations + 1,
|
|
2172
|
+
};
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
/**
|
|
2176
|
+
* Get query optimizer instance.
|
|
2177
|
+
*/
|
|
2178
|
+
getQueryOptimizer(): QueryOptimizer {
|
|
2179
|
+
return this.queryOptimizer;
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
/**
|
|
2183
|
+
* Get index tuner instance.
|
|
2184
|
+
*/
|
|
2185
|
+
getIndexTuner(): IndexTuner {
|
|
2186
|
+
return this.indexTuner;
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
/**
|
|
2190
|
+
* Get pattern recognizer instance.
|
|
2191
|
+
*/
|
|
2192
|
+
getPatternRecognizer(): PatternRecognizer {
|
|
2193
|
+
return this.patternRecognizer;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
/**
|
|
2197
|
+
* Get learning statistics.
|
|
2198
|
+
*/
|
|
2199
|
+
getStats(): LearningStats {
|
|
2200
|
+
return { ...this.learningStats };
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
/**
|
|
2204
|
+
* Get EWC state.
|
|
2205
|
+
*/
|
|
2206
|
+
getEWCState(): EWCState {
|
|
2207
|
+
return {
|
|
2208
|
+
fisherDiagonal: new Map(this.ewcState.fisherDiagonal),
|
|
2209
|
+
previousParams: new Map(this.ewcState.previousParams),
|
|
2210
|
+
consolidationCount: this.ewcState.consolidationCount,
|
|
2211
|
+
lastConsolidation: this.ewcState.lastConsolidation,
|
|
2212
|
+
protectedPatterns: new Set(this.ewcState.protectedPatterns),
|
|
2213
|
+
};
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
/**
|
|
2217
|
+
* Check if learning loop is running.
|
|
2218
|
+
*/
|
|
2219
|
+
isActive(): boolean {
|
|
2220
|
+
return this.isRunning;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
/**
|
|
2224
|
+
* Reset learning state.
|
|
2225
|
+
*/
|
|
2226
|
+
reset(): void {
|
|
2227
|
+
this.stop();
|
|
2228
|
+
this.patternRecognizer.clearPatterns();
|
|
2229
|
+
this.queryOptimizer.clearCache();
|
|
2230
|
+
(this.ewcState.fisherDiagonal as Map<string, number>).clear();
|
|
2231
|
+
(this.ewcState.previousParams as Map<string, number>).clear();
|
|
2232
|
+
(this.ewcState as { consolidationCount: number }).consolidationCount = 0;
|
|
2233
|
+
(this.ewcState.protectedPatterns as Set<string>).clear();
|
|
2234
|
+
this.learningStats = this.initializeStats();
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
// Private helper methods
|
|
2238
|
+
|
|
2239
|
+
private generateFingerprint(sql: string): string {
|
|
2240
|
+
let normalized = sql
|
|
2241
|
+
.replace(/\s+/g, ' ')
|
|
2242
|
+
.replace(/\$\d+/g, '$?')
|
|
2243
|
+
.replace(/'[^']*'/g, "'?'")
|
|
2244
|
+
.replace(/\d+/g, '?')
|
|
2245
|
+
.toLowerCase()
|
|
2246
|
+
.trim();
|
|
2247
|
+
|
|
2248
|
+
let hash = 0;
|
|
2249
|
+
for (let i = 0; i < normalized.length; i++) {
|
|
2250
|
+
const char = normalized.charCodeAt(i);
|
|
2251
|
+
hash = ((hash << 5) - hash) + char;
|
|
2252
|
+
hash = hash & hash;
|
|
2253
|
+
}
|
|
2254
|
+
return `qf_${Math.abs(hash).toString(16)}`;
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
private initializeStats(): LearningStats {
|
|
2258
|
+
return {
|
|
2259
|
+
totalPatterns: 0,
|
|
2260
|
+
activePatterns: 0,
|
|
2261
|
+
expiredPatterns: 0,
|
|
2262
|
+
microLearningEvents: 0,
|
|
2263
|
+
backgroundLearningCycles: 0,
|
|
2264
|
+
ewcConsolidations: 0,
|
|
2265
|
+
avgLearningTimeMs: 0,
|
|
2266
|
+
memoryUsageBytes: 0,
|
|
2267
|
+
lastLearningTimestamp: new Date(),
|
|
2268
|
+
};
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
private updateLearningStats(type: 'micro' | 'background', duration: number): void {
|
|
2272
|
+
const patterns = this.patternRecognizer.getAllPatterns();
|
|
2273
|
+
|
|
2274
|
+
this.learningStats = {
|
|
2275
|
+
totalPatterns: patterns.length,
|
|
2276
|
+
activePatterns: patterns.filter(p => p.confidence > 0.5).length,
|
|
2277
|
+
expiredPatterns: this.learningStats.expiredPatterns,
|
|
2278
|
+
microLearningEvents: this.learningStats.microLearningEvents + (type === 'micro' ? 1 : 0),
|
|
2279
|
+
backgroundLearningCycles: this.learningStats.backgroundLearningCycles + (type === 'background' ? 1 : 0),
|
|
2280
|
+
ewcConsolidations: this.learningStats.ewcConsolidations,
|
|
2281
|
+
avgLearningTimeMs: this.calculateRunningAverage(
|
|
2282
|
+
this.learningStats.avgLearningTimeMs,
|
|
2283
|
+
duration,
|
|
2284
|
+
this.learningStats.microLearningEvents + this.learningStats.backgroundLearningCycles
|
|
2285
|
+
),
|
|
2286
|
+
memoryUsageBytes: this.estimateMemoryUsage(),
|
|
2287
|
+
lastLearningTimestamp: new Date(),
|
|
2288
|
+
};
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
private calculateRunningAverage(currentAvg: number, newValue: number, count: number): number {
|
|
2292
|
+
if (count === 0) return newValue;
|
|
2293
|
+
return (currentAvg * count + newValue) / (count + 1);
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
private estimateMemoryUsage(): number {
|
|
2297
|
+
// Rough estimation of memory usage
|
|
2298
|
+
const patterns = this.patternRecognizer.getAllPatterns();
|
|
2299
|
+
const patternBytes = patterns.length * 500; // ~500 bytes per pattern
|
|
2300
|
+
const queryStatsBytes = (this.queryOptimizer.getQueryStats() as QueryExecutionStats[]).length * 200;
|
|
2301
|
+
const ewcBytes = this.ewcState.fisherDiagonal.size * 32;
|
|
2302
|
+
|
|
2303
|
+
return patternBytes + queryStatsBytes + ewcBytes;
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
// ============================================================================
|
|
2308
|
+
// Factory and Exports
|
|
2309
|
+
// ============================================================================
|
|
2310
|
+
|
|
2311
|
+
/**
|
|
2312
|
+
* Create a complete self-learning system.
|
|
2313
|
+
*/
|
|
2314
|
+
export function createSelfLearningSystem(config?: Partial<LearningConfig>): {
|
|
2315
|
+
learningLoop: LearningLoop;
|
|
2316
|
+
queryOptimizer: QueryOptimizer;
|
|
2317
|
+
indexTuner: IndexTuner;
|
|
2318
|
+
patternRecognizer: PatternRecognizer;
|
|
2319
|
+
} {
|
|
2320
|
+
const learningLoop = new LearningLoop(config);
|
|
2321
|
+
|
|
2322
|
+
return {
|
|
2323
|
+
learningLoop,
|
|
2324
|
+
queryOptimizer: learningLoop.getQueryOptimizer(),
|
|
2325
|
+
indexTuner: learningLoop.getIndexTuner(),
|
|
2326
|
+
patternRecognizer: learningLoop.getPatternRecognizer(),
|
|
2327
|
+
};
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
/**
|
|
2331
|
+
* Default configuration for production use.
|
|
2332
|
+
*/
|
|
2333
|
+
export const DEFAULT_LEARNING_CONFIG: LearningConfig = {
|
|
2334
|
+
enableMicroLearning: true,
|
|
2335
|
+
microLearningThresholdMs: 0.1,
|
|
2336
|
+
enableBackgroundLearning: true,
|
|
2337
|
+
backgroundLearningIntervalMs: 60000,
|
|
2338
|
+
enableEWC: true,
|
|
2339
|
+
ewcLambda: 0.5,
|
|
2340
|
+
maxPatterns: 10000,
|
|
2341
|
+
patternExpiryMs: 86400000,
|
|
2342
|
+
learningRate: 0.01,
|
|
2343
|
+
momentum: 0.9,
|
|
2344
|
+
};
|
|
2345
|
+
|
|
2346
|
+
/**
|
|
2347
|
+
* High-performance configuration (less learning, more speed).
|
|
2348
|
+
*/
|
|
2349
|
+
export const HIGH_PERF_LEARNING_CONFIG: LearningConfig = {
|
|
2350
|
+
enableMicroLearning: false,
|
|
2351
|
+
microLearningThresholdMs: 0,
|
|
2352
|
+
enableBackgroundLearning: true,
|
|
2353
|
+
backgroundLearningIntervalMs: 300000, // 5 minutes
|
|
2354
|
+
enableEWC: false,
|
|
2355
|
+
ewcLambda: 0,
|
|
2356
|
+
maxPatterns: 1000,
|
|
2357
|
+
patternExpiryMs: 3600000, // 1 hour
|
|
2358
|
+
learningRate: 0.001,
|
|
2359
|
+
momentum: 0.5,
|
|
2360
|
+
};
|
|
2361
|
+
|
|
2362
|
+
/**
|
|
2363
|
+
* High-accuracy configuration (more learning, potentially slower).
|
|
2364
|
+
*/
|
|
2365
|
+
export const HIGH_ACCURACY_LEARNING_CONFIG: LearningConfig = {
|
|
2366
|
+
enableMicroLearning: true,
|
|
2367
|
+
microLearningThresholdMs: 0.05,
|
|
2368
|
+
enableBackgroundLearning: true,
|
|
2369
|
+
backgroundLearningIntervalMs: 30000, // 30 seconds
|
|
2370
|
+
enableEWC: true,
|
|
2371
|
+
ewcLambda: 0.8,
|
|
2372
|
+
maxPatterns: 50000,
|
|
2373
|
+
patternExpiryMs: 604800000, // 7 days
|
|
2374
|
+
learningRate: 0.05,
|
|
2375
|
+
momentum: 0.95,
|
|
2376
|
+
};
|