@sparkleideas/plugins 3.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +401 -0
  2. package/__tests__/collection-manager.test.ts +332 -0
  3. package/__tests__/dependency-graph.test.ts +434 -0
  4. package/__tests__/enhanced-plugin-registry.test.ts +488 -0
  5. package/__tests__/plugin-registry.test.ts +368 -0
  6. package/__tests__/ruvector-bridge.test.ts +2429 -0
  7. package/__tests__/ruvector-integration.test.ts +1602 -0
  8. package/__tests__/ruvector-migrations.test.ts +1099 -0
  9. package/__tests__/ruvector-quantization.test.ts +846 -0
  10. package/__tests__/ruvector-streaming.test.ts +1088 -0
  11. package/__tests__/sdk.test.ts +325 -0
  12. package/__tests__/security.test.ts +348 -0
  13. package/__tests__/utils/ruvector-test-utils.ts +860 -0
  14. package/examples/plugin-creator/index.ts +636 -0
  15. package/examples/plugin-creator/plugin-creator.test.ts +312 -0
  16. package/examples/ruvector/README.md +288 -0
  17. package/examples/ruvector/attention-patterns.ts +394 -0
  18. package/examples/ruvector/basic-usage.ts +288 -0
  19. package/examples/ruvector/docker-compose.yml +75 -0
  20. package/examples/ruvector/gnn-analysis.ts +501 -0
  21. package/examples/ruvector/hyperbolic-hierarchies.ts +557 -0
  22. package/examples/ruvector/init-db.sql +119 -0
  23. package/examples/ruvector/quantization.ts +680 -0
  24. package/examples/ruvector/self-learning.ts +447 -0
  25. package/examples/ruvector/semantic-search.ts +576 -0
  26. package/examples/ruvector/streaming-large-data.ts +507 -0
  27. package/examples/ruvector/transactions.ts +594 -0
  28. package/examples/ruvector-plugins/hook-pattern-library.ts +486 -0
  29. package/examples/ruvector-plugins/index.ts +79 -0
  30. package/examples/ruvector-plugins/intent-router.ts +354 -0
  31. package/examples/ruvector-plugins/mcp-tool-optimizer.ts +424 -0
  32. package/examples/ruvector-plugins/reasoning-bank.ts +657 -0
  33. package/examples/ruvector-plugins/ruvector-plugins.test.ts +518 -0
  34. package/examples/ruvector-plugins/semantic-code-search.ts +498 -0
  35. package/examples/ruvector-plugins/shared/index.ts +20 -0
  36. package/examples/ruvector-plugins/shared/vector-utils.ts +257 -0
  37. package/examples/ruvector-plugins/sona-learning.ts +445 -0
  38. package/package.json +97 -0
  39. package/src/collections/collection-manager.ts +661 -0
  40. package/src/collections/index.ts +56 -0
  41. package/src/collections/official/index.ts +1040 -0
  42. package/src/core/base-plugin.ts +416 -0
  43. package/src/core/plugin-interface.ts +215 -0
  44. package/src/hooks/index.ts +685 -0
  45. package/src/index.ts +378 -0
  46. package/src/integrations/agentic-flow.ts +743 -0
  47. package/src/integrations/index.ts +88 -0
  48. package/src/integrations/ruvector/ARCHITECTURE.md +1245 -0
  49. package/src/integrations/ruvector/attention-advanced.ts +1040 -0
  50. package/src/integrations/ruvector/attention-executor.ts +782 -0
  51. package/src/integrations/ruvector/attention-mechanisms.ts +757 -0
  52. package/src/integrations/ruvector/attention.ts +1063 -0
  53. package/src/integrations/ruvector/gnn.ts +3050 -0
  54. package/src/integrations/ruvector/hyperbolic.ts +1948 -0
  55. package/src/integrations/ruvector/index.ts +394 -0
  56. package/src/integrations/ruvector/migrations/001_create_extension.sql +135 -0
  57. package/src/integrations/ruvector/migrations/002_create_vector_tables.sql +259 -0
  58. package/src/integrations/ruvector/migrations/003_create_indices.sql +328 -0
  59. package/src/integrations/ruvector/migrations/004_create_functions.sql +598 -0
  60. package/src/integrations/ruvector/migrations/005_create_attention_functions.sql +654 -0
  61. package/src/integrations/ruvector/migrations/006_create_gnn_functions.sql +728 -0
  62. package/src/integrations/ruvector/migrations/007_create_hyperbolic_functions.sql +762 -0
  63. package/src/integrations/ruvector/migrations/index.ts +35 -0
  64. package/src/integrations/ruvector/migrations/migrations.ts +647 -0
  65. package/src/integrations/ruvector/quantization.ts +2036 -0
  66. package/src/integrations/ruvector/ruvector-bridge.ts +2000 -0
  67. package/src/integrations/ruvector/self-learning.ts +2376 -0
  68. package/src/integrations/ruvector/streaming.ts +1737 -0
  69. package/src/integrations/ruvector/types.ts +1945 -0
  70. package/src/providers/index.ts +643 -0
  71. package/src/registry/dependency-graph.ts +568 -0
  72. package/src/registry/enhanced-plugin-registry.ts +994 -0
  73. package/src/registry/plugin-registry.ts +604 -0
  74. package/src/sdk/index.ts +563 -0
  75. package/src/security/index.ts +594 -0
  76. package/src/types/index.ts +446 -0
  77. package/src/workers/index.ts +700 -0
  78. package/tmp.json +0 -0
  79. package/tsconfig.json +25 -0
  80. package/vitest.config.ts +23 -0
@@ -0,0 +1,2376 @@
1
+ /**
2
+ * RuVector Self-Learning Optimization Module
3
+ *
4
+ * SONA-inspired self-learning features for the RuVector PostgreSQL Bridge.
5
+ * Implements adaptive query optimization, intelligent index tuning,
6
+ * pattern recognition, and continuous learning with EWC++ protection.
7
+ *
8
+ * @module @sparkleideas/plugins/integrations/ruvector/self-learning
9
+ * @version 1.0.0
10
+ */
11
+
12
+ import type {
13
+ VectorIndexType,
14
+ DistanceMetric,
15
+ IndexStats,
16
+ QueryStats,
17
+ VectorSearchOptions,
18
+ } from './types.js';
19
+
20
+ // ============================================================================
21
+ // Query Analysis Types
22
+ // ============================================================================
23
+
24
+ /**
25
+ * Analysis result for a SQL query.
26
+ */
27
+ export interface QueryAnalysis {
28
+ /** Original SQL query */
29
+ readonly sql: string;
30
+ /** Query type (SELECT, INSERT, UPDATE, DELETE) */
31
+ readonly queryType: QueryType;
32
+ /** Tables referenced in the query */
33
+ readonly tables: string[];
34
+ /** Columns referenced in the query */
35
+ readonly columns: string[];
36
+ /** Vector operations detected */
37
+ readonly vectorOperations: VectorOperation[];
38
+ /** Estimated complexity score (0-1) */
39
+ readonly complexity: number;
40
+ /** Index usage hints */
41
+ readonly indexHints: IndexHint[];
42
+ /** Potential bottlenecks */
43
+ readonly bottlenecks: Bottleneck[];
44
+ /** Parse time in milliseconds */
45
+ readonly parseTimeMs: number;
46
+ /** Query fingerprint for deduplication */
47
+ readonly fingerprint: string;
48
+ }
49
+
50
+ /**
51
+ * Query types supported.
52
+ */
53
+ export type QueryType = 'SELECT' | 'INSERT' | 'UPDATE' | 'DELETE' | 'UNKNOWN';
54
+
55
+ /**
56
+ * Vector operation in a query.
57
+ */
58
+ export interface VectorOperation {
59
+ /** Operation type */
60
+ readonly type: 'search' | 'insert' | 'update' | 'aggregate' | 'distance';
61
+ /** Table name */
62
+ readonly table: string;
63
+ /** Column name */
64
+ readonly column: string;
65
+ /** Distance metric used */
66
+ readonly metric?: DistanceMetric;
67
+ /** K value for KNN */
68
+ readonly k?: number;
69
+ /** Estimated cost */
70
+ readonly estimatedCost: number;
71
+ }
72
+
73
+ /**
74
+ * Index usage hint.
75
+ */
76
+ export interface IndexHint {
77
+ /** Recommended index type */
78
+ readonly indexType: VectorIndexType;
79
+ /** Table name */
80
+ readonly table: string;
81
+ /** Column name */
82
+ readonly column: string;
83
+ /** Confidence score (0-1) */
84
+ readonly confidence: number;
85
+ /** Expected speedup factor */
86
+ readonly expectedSpeedup: number;
87
+ }
88
+
89
+ /**
90
+ * Query bottleneck.
91
+ */
92
+ export interface Bottleneck {
93
+ /** Bottleneck type */
94
+ readonly type: 'full_scan' | 'missing_index' | 'cartesian_product' | 'large_sort' | 'expensive_function';
95
+ /** Description */
96
+ readonly description: string;
97
+ /** Severity (1-10) */
98
+ readonly severity: number;
99
+ /** Suggested fix */
100
+ readonly suggestion: string;
101
+ }
102
+
103
+ /**
104
+ * Query optimization suggestion.
105
+ */
106
+ export interface Optimization {
107
+ /** Optimization type */
108
+ readonly type: OptimizationType;
109
+ /** Description of the optimization */
110
+ readonly description: string;
111
+ /** Original query fragment */
112
+ readonly original: string;
113
+ /** Optimized query fragment */
114
+ readonly optimized: string;
115
+ /** Expected improvement percentage */
116
+ readonly expectedImprovement: number;
117
+ /** Confidence score (0-1) */
118
+ readonly confidence: number;
119
+ /** Risk level */
120
+ readonly risk: 'low' | 'medium' | 'high';
121
+ /** Apply automatically */
122
+ readonly autoApply: boolean;
123
+ }
124
+
125
+ /**
126
+ * Types of query optimizations.
127
+ */
128
+ export type OptimizationType =
129
+ | 'index_usage'
130
+ | 'query_rewrite'
131
+ | 'parameter_tuning'
132
+ | 'caching'
133
+ | 'batching'
134
+ | 'projection_pushdown'
135
+ | 'filter_pushdown'
136
+ | 'limit_pushdown'
137
+ | 'parallel_execution';
138
+
139
+ /**
140
+ * Query execution statistics.
141
+ */
142
+ export interface QueryExecutionStats {
143
+ /** Query fingerprint */
144
+ readonly fingerprint: string;
145
+ /** SQL query */
146
+ readonly sql: string;
147
+ /** Execution count */
148
+ readonly executionCount: number;
149
+ /** Total execution time (ms) */
150
+ readonly totalDurationMs: number;
151
+ /** Average execution time (ms) */
152
+ readonly avgDurationMs: number;
153
+ /** Min execution time (ms) */
154
+ readonly minDurationMs: number;
155
+ /** Max execution time (ms) */
156
+ readonly maxDurationMs: number;
157
+ /** P95 execution time (ms) */
158
+ readonly p95DurationMs: number;
159
+ /** P99 execution time (ms) */
160
+ readonly p99DurationMs: number;
161
+ /** Total rows returned */
162
+ readonly totalRows: number;
163
+ /** Average rows per execution */
164
+ readonly avgRows: number;
165
+ /** Last executed timestamp */
166
+ readonly lastExecuted: Date;
167
+ /** First executed timestamp */
168
+ readonly firstExecuted: Date;
169
+ /** Error count */
170
+ readonly errorCount: number;
171
+ }
172
+
173
+ // ============================================================================
174
+ // Index Tuning Types
175
+ // ============================================================================
176
+
177
+ /**
178
+ * Workload analysis result.
179
+ */
180
+ export interface WorkloadAnalysis {
181
+ /** Analysis timestamp */
182
+ readonly timestamp: Date;
183
+ /** Analysis duration (ms) */
184
+ readonly durationMs: number;
185
+ /** Total queries analyzed */
186
+ readonly totalQueries: number;
187
+ /** Query type distribution */
188
+ readonly queryDistribution: Map<QueryType, number>;
189
+ /** Most frequent query patterns */
190
+ readonly topPatterns: QueryPattern[];
191
+ /** Hot tables (most accessed) */
192
+ readonly hotTables: TableAccess[];
193
+ /** Index usage summary */
194
+ readonly indexUsage: IndexUsageSummary[];
195
+ /** Workload characteristics */
196
+ readonly characteristics: WorkloadCharacteristics;
197
+ /** Recommendations */
198
+ readonly recommendations: WorkloadRecommendation[];
199
+ }
200
+
201
+ /**
202
+ * Query pattern from workload analysis.
203
+ */
204
+ export interface QueryPattern {
205
+ /** Pattern fingerprint */
206
+ readonly fingerprint: string;
207
+ /** Example query */
208
+ readonly example: string;
209
+ /** Execution frequency */
210
+ readonly frequency: number;
211
+ /** Average duration (ms) */
212
+ readonly avgDurationMs: number;
213
+ /** Tables involved */
214
+ readonly tables: string[];
215
+ /** Is vector search */
216
+ readonly isVectorSearch: boolean;
217
+ }
218
+
219
+ /**
220
+ * Table access statistics.
221
+ */
222
+ export interface TableAccess {
223
+ /** Table name */
224
+ readonly tableName: string;
225
+ /** Read count */
226
+ readonly reads: number;
227
+ /** Write count */
228
+ readonly writes: number;
229
+ /** Vector search count */
230
+ readonly vectorSearches: number;
231
+ /** Average scan size */
232
+ readonly avgScanSize: number;
233
+ /** Is frequently accessed */
234
+ readonly isHot: boolean;
235
+ }
236
+
237
+ /**
238
+ * Index usage summary.
239
+ */
240
+ export interface IndexUsageSummary {
241
+ /** Index name */
242
+ readonly indexName: string;
243
+ /** Table name */
244
+ readonly tableName: string;
245
+ /** Index type */
246
+ readonly indexType: VectorIndexType;
247
+ /** Scan count */
248
+ readonly scanCount: number;
249
+ /** Tuple reads */
250
+ readonly tupleReads: number;
251
+ /** Tuple fetches */
252
+ readonly tupleFetches: number;
253
+ /** Is underutilized */
254
+ readonly isUnderutilized: boolean;
255
+ /** Recommendation */
256
+ readonly recommendation: 'keep' | 'drop' | 'rebuild' | 'tune';
257
+ }
258
+
259
+ /**
260
+ * Workload characteristics.
261
+ */
262
+ export interface WorkloadCharacteristics {
263
+ /** Read/write ratio */
264
+ readonly readWriteRatio: number;
265
+ /** Vector search percentage */
266
+ readonly vectorSearchPercentage: number;
267
+ /** Average query complexity */
268
+ readonly avgComplexity: number;
269
+ /** Peak hours (0-23) */
270
+ readonly peakHours: number[];
271
+ /** Is OLTP-like */
272
+ readonly isOLTP: boolean;
273
+ /** Is OLAP-like */
274
+ readonly isOLAP: boolean;
275
+ /** Is hybrid */
276
+ readonly isHybrid: boolean;
277
+ }
278
+
279
+ /**
280
+ * Workload-based recommendation.
281
+ */
282
+ export interface WorkloadRecommendation {
283
+ /** Recommendation type */
284
+ readonly type: 'create_index' | 'drop_index' | 'tune_parameter' | 'partition_table' | 'materialize_view';
285
+ /** Priority (1-10) */
286
+ readonly priority: number;
287
+ /** Description */
288
+ readonly description: string;
289
+ /** Estimated impact */
290
+ readonly estimatedImpact: string;
291
+ /** SQL to execute */
292
+ readonly sql?: string;
293
+ }
294
+
295
+ /**
296
+ * Index suggestion.
297
+ */
298
+ export interface IndexSuggestion {
299
+ /** Table name */
300
+ readonly tableName: string;
301
+ /** Column name */
302
+ readonly columnName: string;
303
+ /** Suggested index type */
304
+ readonly indexType: VectorIndexType;
305
+ /** Suggested index name */
306
+ readonly indexName: string;
307
+ /** Distance metric */
308
+ readonly metric?: DistanceMetric;
309
+ /** HNSW M parameter */
310
+ readonly m?: number;
311
+ /** HNSW ef_construction */
312
+ readonly efConstruction?: number;
313
+ /** IVF lists */
314
+ readonly lists?: number;
315
+ /** Confidence score (0-1) */
316
+ readonly confidence: number;
317
+ /** Expected improvement */
318
+ readonly expectedImprovement: number;
319
+ /** Rationale */
320
+ readonly rationale: string;
321
+ /** CREATE INDEX SQL */
322
+ readonly createSql: string;
323
+ }
324
+
325
+ /**
326
+ * HNSW parameters.
327
+ */
328
+ export interface HNSWParams {
329
+ /** M parameter (connections per layer) */
330
+ readonly m: number;
331
+ /** ef_construction parameter */
332
+ readonly efConstruction: number;
333
+ /** ef_search parameter */
334
+ readonly efSearch: number;
335
+ /** Optimal for workload */
336
+ readonly optimizedFor: 'recall' | 'speed' | 'balanced';
337
+ /** Tuning confidence (0-1) */
338
+ readonly confidence: number;
339
+ /** Estimated recall */
340
+ readonly estimatedRecall: number;
341
+ /** Estimated QPS */
342
+ readonly estimatedQps: number;
343
+ }
344
+
345
+ // ============================================================================
346
+ // Pattern Recognition Types
347
+ // ============================================================================
348
+
349
+ /**
350
+ * Query history entry.
351
+ */
352
+ export interface QueryHistory {
353
+ /** Query fingerprint */
354
+ readonly fingerprint: string;
355
+ /** SQL query */
356
+ readonly sql: string;
357
+ /** Execution timestamp */
358
+ readonly timestamp: Date;
359
+ /** Duration (ms) */
360
+ readonly durationMs: number;
361
+ /** Rows returned */
362
+ readonly rowCount: number;
363
+ /** Was successful */
364
+ readonly success: boolean;
365
+ /** User/session ID */
366
+ readonly sessionId?: string;
367
+ /** Context metadata */
368
+ readonly context?: Record<string, unknown>;
369
+ }
370
+
371
+ /**
372
+ * Detected query pattern.
373
+ */
374
+ export interface Pattern {
375
+ /** Pattern ID */
376
+ readonly id: string;
377
+ /** Pattern type */
378
+ readonly type: PatternType;
379
+ /** Pattern signature */
380
+ readonly signature: string;
381
+ /** Description */
382
+ readonly description: string;
383
+ /** Confidence score (0-1) */
384
+ readonly confidence: number;
385
+ /** Occurrence count */
386
+ readonly occurrences: number;
387
+ /** Example queries matching this pattern */
388
+ readonly examples: string[];
389
+ /** Temporal characteristics */
390
+ readonly temporal?: TemporalPattern;
391
+ /** Performance characteristics */
392
+ readonly performance: PerformancePattern;
393
+ /** First detected */
394
+ readonly firstDetected: Date;
395
+ /** Last detected */
396
+ readonly lastDetected: Date;
397
+ }
398
+
399
+ /**
400
+ * Pattern types.
401
+ */
402
+ export type PatternType =
403
+ | 'sequential_access'
404
+ | 'random_access'
405
+ | 'bulk_insert'
406
+ | 'bulk_update'
407
+ | 'similarity_search'
408
+ | 'range_query'
409
+ | 'aggregation'
410
+ | 'join_pattern'
411
+ | 'periodic'
412
+ | 'burst'
413
+ | 'degrading_performance';
414
+
415
+ /**
416
+ * Temporal pattern characteristics.
417
+ */
418
+ export interface TemporalPattern {
419
+ /** Is periodic */
420
+ readonly isPeriodic: boolean;
421
+ /** Period in seconds (if periodic) */
422
+ readonly periodSeconds?: number;
423
+ /** Peak times (hour of day) */
424
+ readonly peakHours: number[];
425
+ /** Trend direction */
426
+ readonly trend: 'increasing' | 'decreasing' | 'stable' | 'volatile';
427
+ /** Seasonality detected */
428
+ readonly hasSeasonality: boolean;
429
+ }
430
+
431
+ /**
432
+ * Performance pattern.
433
+ */
434
+ export interface PerformancePattern {
435
+ /** Average response time trend */
436
+ readonly responseTrend: 'improving' | 'degrading' | 'stable';
437
+ /** Variance coefficient */
438
+ readonly varianceCoefficient: number;
439
+ /** Has outliers */
440
+ readonly hasOutliers: boolean;
441
+ /** Percentile distribution */
442
+ readonly percentiles: {
443
+ readonly p50: number;
444
+ readonly p75: number;
445
+ readonly p90: number;
446
+ readonly p95: number;
447
+ readonly p99: number;
448
+ };
449
+ }
450
+
451
+ /**
452
+ * Query prediction context.
453
+ */
454
+ export interface Context {
455
+ /** Current session ID */
456
+ readonly sessionId?: string;
457
+ /** Recent query fingerprints */
458
+ readonly recentQueries: string[];
459
+ /** Current time */
460
+ readonly timestamp: Date;
461
+ /** User context */
462
+ readonly userContext?: Record<string, unknown>;
463
+ /** Application context */
464
+ readonly appContext?: Record<string, unknown>;
465
+ }
466
+
467
+ /**
468
+ * Query anomaly.
469
+ */
470
+ export interface Anomaly {
471
+ /** Anomaly ID */
472
+ readonly id: string;
473
+ /** Anomaly type */
474
+ readonly type: AnomalyType;
475
+ /** Affected query */
476
+ readonly query: string;
477
+ /** Query fingerprint */
478
+ readonly fingerprint: string;
479
+ /** Detection timestamp */
480
+ readonly timestamp: Date;
481
+ /** Severity (1-10) */
482
+ readonly severity: number;
483
+ /** Description */
484
+ readonly description: string;
485
+ /** Expected value */
486
+ readonly expected: number;
487
+ /** Actual value */
488
+ readonly actual: number;
489
+ /** Deviation from normal */
490
+ readonly deviation: number;
491
+ /** Possible causes */
492
+ readonly possibleCauses: string[];
493
+ /** Recommended actions */
494
+ readonly recommendations: string[];
495
+ }
496
+
497
+ /**
498
+ * Anomaly types.
499
+ */
500
+ export type AnomalyType =
501
+ | 'slow_query'
502
+ | 'high_resource_usage'
503
+ | 'unusual_pattern'
504
+ | 'error_spike'
505
+ | 'traffic_anomaly'
506
+ | 'data_drift'
507
+ | 'index_degradation'
508
+ | 'cardinality_change';
509
+
510
+ // ============================================================================
511
+ // Learning System Types
512
+ // ============================================================================
513
+
514
+ /**
515
+ * Learning configuration.
516
+ */
517
+ export interface LearningConfig {
518
+ /** Enable micro-learning */
519
+ readonly enableMicroLearning: boolean;
520
+ /** Micro-learning threshold (ms) */
521
+ readonly microLearningThresholdMs: number;
522
+ /** Enable background learning */
523
+ readonly enableBackgroundLearning: boolean;
524
+ /** Background learning interval (ms) */
525
+ readonly backgroundLearningIntervalMs: number;
526
+ /** Enable EWC++ */
527
+ readonly enableEWC: boolean;
528
+ /** EWC lambda (regularization strength) */
529
+ readonly ewcLambda: number;
530
+ /** Maximum patterns to retain */
531
+ readonly maxPatterns: number;
532
+ /** Pattern expiry time (ms) */
533
+ readonly patternExpiryMs: number;
534
+ /** Learning rate */
535
+ readonly learningRate: number;
536
+ /** Momentum */
537
+ readonly momentum: number;
538
+ }
539
+
540
+ /**
541
+ * Learning statistics.
542
+ */
543
+ export interface LearningStats {
544
+ /** Total patterns learned */
545
+ readonly totalPatterns: number;
546
+ /** Active patterns */
547
+ readonly activePatterns: number;
548
+ /** Expired patterns */
549
+ readonly expiredPatterns: number;
550
+ /** Micro-learning events */
551
+ readonly microLearningEvents: number;
552
+ /** Background learning cycles */
553
+ readonly backgroundLearningCycles: number;
554
+ /** EWC consolidations */
555
+ readonly ewcConsolidations: number;
556
+ /** Average learning time (ms) */
557
+ readonly avgLearningTimeMs: number;
558
+ /** Memory usage (bytes) */
559
+ readonly memoryUsageBytes: number;
560
+ /** Last learning timestamp */
561
+ readonly lastLearningTimestamp: Date;
562
+ }
563
+
564
+ /**
565
+ * EWC++ state for preventing catastrophic forgetting.
566
+ */
567
+ export interface EWCState {
568
+ /** Fisher information matrix (diagonal approximation) */
569
+ readonly fisherDiagonal: Map<string, number>;
570
+ /** Previous parameter values */
571
+ readonly previousParams: Map<string, number>;
572
+ /** Consolidation count */
573
+ readonly consolidationCount: number;
574
+ /** Last consolidation timestamp */
575
+ readonly lastConsolidation: Date;
576
+ /** Protected patterns */
577
+ readonly protectedPatterns: Set<string>;
578
+ }
579
+
580
+ // ============================================================================
581
+ // Query Optimizer Implementation
582
+ // ============================================================================
583
+
584
+ /**
585
+ * Query Optimizer for analyzing and optimizing SQL queries.
586
+ * Implements SONA-inspired micro-learning for real-time adaptation.
587
+ */
588
+ export class QueryOptimizer {
589
+ private readonly queryStats: Map<string, QueryExecutionStats> = new Map();
590
+ private readonly optimizationCache: Map<string, Optimization[]> = new Map();
591
+ private readonly config: LearningConfig;
592
+
593
+ constructor(config?: Partial<LearningConfig>) {
594
+ this.config = {
595
+ enableMicroLearning: true,
596
+ microLearningThresholdMs: 0.1, // <0.1ms for micro-learning
597
+ enableBackgroundLearning: true,
598
+ backgroundLearningIntervalMs: 60000,
599
+ enableEWC: true,
600
+ ewcLambda: 0.5,
601
+ maxPatterns: 10000,
602
+ patternExpiryMs: 86400000, // 24 hours
603
+ learningRate: 0.01,
604
+ momentum: 0.9,
605
+ ...config,
606
+ };
607
+ }
608
+
609
+ /**
610
+ * Analyze a SQL query and return detailed analysis.
611
+ */
612
+ analyzeQuery(sql: string): QueryAnalysis {
613
+ const startTime = performance.now();
614
+
615
+ // Parse query type
616
+ const queryType = this.parseQueryType(sql);
617
+
618
+ // Extract tables
619
+ const tables = this.extractTables(sql);
620
+
621
+ // Extract columns
622
+ const columns = this.extractColumns(sql);
623
+
624
+ // Detect vector operations
625
+ const vectorOperations = this.detectVectorOperations(sql, tables);
626
+
627
+ // Calculate complexity
628
+ const complexity = this.calculateComplexity(sql, vectorOperations);
629
+
630
+ // Generate index hints
631
+ const indexHints = this.generateIndexHints(sql, tables, vectorOperations);
632
+
633
+ // Detect bottlenecks
634
+ const bottlenecks = this.detectBottlenecks(sql, tables, vectorOperations);
635
+
636
+ // Generate fingerprint
637
+ const fingerprint = this.generateFingerprint(sql);
638
+
639
+ const parseTimeMs = performance.now() - startTime;
640
+
641
+ return {
642
+ sql,
643
+ queryType,
644
+ tables,
645
+ columns,
646
+ vectorOperations,
647
+ complexity,
648
+ indexHints,
649
+ bottlenecks,
650
+ parseTimeMs,
651
+ fingerprint,
652
+ };
653
+ }
654
+
655
+ /**
656
+ * Suggest optimizations for a query analysis.
657
+ */
658
+ suggestOptimizations(analysis: QueryAnalysis): Optimization[] {
659
+ // Check cache first
660
+ const cached = this.optimizationCache.get(analysis.fingerprint);
661
+ if (cached) {
662
+ return cached;
663
+ }
664
+
665
+ const optimizations: Optimization[] = [];
666
+
667
+ // Index usage optimizations
668
+ for (const hint of analysis.indexHints) {
669
+ if (hint.confidence > 0.7) {
670
+ optimizations.push({
671
+ type: 'index_usage',
672
+ description: `Create ${hint.indexType} index on ${hint.table}.${hint.column}`,
673
+ original: '',
674
+ optimized: `CREATE INDEX idx_${hint.table}_${hint.column} ON ${hint.table} USING ${hint.indexType} (${hint.column})`,
675
+ expectedImprovement: hint.expectedSpeedup * 100,
676
+ confidence: hint.confidence,
677
+ risk: 'low',
678
+ autoApply: false,
679
+ });
680
+ }
681
+ }
682
+
683
+ // Vector search optimizations
684
+ for (const op of analysis.vectorOperations) {
685
+ if (op.type === 'search' && op.estimatedCost > 100) {
686
+ optimizations.push({
687
+ type: 'parameter_tuning',
688
+ description: `Tune ef_search for ${op.table}.${op.column} vector search`,
689
+ original: '',
690
+ optimized: `SET hnsw.ef_search = ${Math.min(op.k! * 4, 200)}`,
691
+ expectedImprovement: 30,
692
+ confidence: 0.8,
693
+ risk: 'low',
694
+ autoApply: true,
695
+ });
696
+ }
697
+ }
698
+
699
+ // Query rewrite optimizations
700
+ if (analysis.bottlenecks.some(b => b.type === 'full_scan')) {
701
+ optimizations.push({
702
+ type: 'query_rewrite',
703
+ description: 'Add LIMIT clause to prevent full table scan',
704
+ original: analysis.sql,
705
+ optimized: analysis.sql.includes('LIMIT') ? analysis.sql : `${analysis.sql} LIMIT 1000`,
706
+ expectedImprovement: 50,
707
+ confidence: 0.6,
708
+ risk: 'medium',
709
+ autoApply: false,
710
+ });
711
+ }
712
+
713
+ // Batching optimizations for multiple inserts
714
+ if (analysis.queryType === 'INSERT' && analysis.complexity > 0.5) {
715
+ optimizations.push({
716
+ type: 'batching',
717
+ description: 'Use batch insert for better performance',
718
+ original: analysis.sql,
719
+ optimized: 'Use COPY or multi-row INSERT',
720
+ expectedImprovement: 80,
721
+ confidence: 0.9,
722
+ risk: 'low',
723
+ autoApply: false,
724
+ });
725
+ }
726
+
727
+ // Projection pushdown
728
+ if (analysis.sql.includes('SELECT *')) {
729
+ const neededColumns = analysis.columns.slice(0, 5).join(', ');
730
+ optimizations.push({
731
+ type: 'projection_pushdown',
732
+ description: 'Select only needed columns instead of SELECT *',
733
+ original: 'SELECT *',
734
+ optimized: `SELECT ${neededColumns || 'id, ...needed_columns'}`,
735
+ expectedImprovement: 20,
736
+ confidence: 0.85,
737
+ risk: 'low',
738
+ autoApply: false,
739
+ });
740
+ }
741
+
742
+ // Cache the results
743
+ this.optimizationCache.set(analysis.fingerprint, optimizations);
744
+
745
+ return optimizations;
746
+ }
747
+
748
+ /**
749
+ * Rewrite a query for better performance.
750
+ */
751
+ rewriteQuery(sql: string): string {
752
+ let rewritten = sql.trim();
753
+
754
+ // Normalize whitespace
755
+ rewritten = rewritten.replace(/\s+/g, ' ');
756
+
757
+ // Add missing semicolon
758
+ if (!rewritten.endsWith(';')) {
759
+ rewritten += ';';
760
+ }
761
+
762
+ // Optimize ORDER BY with LIMIT
763
+ const orderLimitMatch = rewritten.match(/ORDER BY\s+([^\s]+)\s+(ASC|DESC)?\s*;$/i);
764
+ if (orderLimitMatch && !rewritten.includes('LIMIT')) {
765
+ rewritten = rewritten.replace(/;$/, ' LIMIT 100;');
766
+ }
767
+
768
+ // Optimize vector distance calculations
769
+ rewritten = rewritten.replace(
770
+ /(\w+)\s*<->\s*\$\d+/g,
771
+ (match, column) => `${column} <=> $1` // Use cosine for better cache locality
772
+ );
773
+
774
+ // Add EXPLAIN ANALYZE for slow queries (for debugging)
775
+ // This is disabled in production
776
+ // rewritten = `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${rewritten}`;
777
+
778
+ return rewritten;
779
+ }
780
+
781
+ /**
782
+ * Record query execution statistics for learning.
783
+ */
784
+ recordQueryStats(query: string, duration: number, rows: number): void {
785
+ const fingerprint = this.generateFingerprint(query);
786
+ const existing = this.queryStats.get(fingerprint);
787
+ const now = new Date();
788
+
789
+ if (existing) {
790
+ // Update existing stats
791
+ const newCount = existing.executionCount + 1;
792
+ const newTotalDuration = existing.totalDurationMs + duration;
793
+ const newTotalRows = existing.totalRows + rows;
794
+
795
+ // Update percentiles (simplified - in production use a proper algorithm)
796
+ const durations = [existing.avgDurationMs * existing.executionCount, duration];
797
+ durations.sort((a, b) => a - b);
798
+
799
+ this.queryStats.set(fingerprint, {
800
+ fingerprint,
801
+ sql: query,
802
+ executionCount: newCount,
803
+ totalDurationMs: newTotalDuration,
804
+ avgDurationMs: newTotalDuration / newCount,
805
+ minDurationMs: Math.min(existing.minDurationMs, duration),
806
+ maxDurationMs: Math.max(existing.maxDurationMs, duration),
807
+ p95DurationMs: this.calculatePercentile(durations, 0.95),
808
+ p99DurationMs: this.calculatePercentile(durations, 0.99),
809
+ totalRows: newTotalRows,
810
+ avgRows: newTotalRows / newCount,
811
+ lastExecuted: now,
812
+ firstExecuted: existing.firstExecuted,
813
+ errorCount: existing.errorCount,
814
+ });
815
+ } else {
816
+ // Create new stats
817
+ this.queryStats.set(fingerprint, {
818
+ fingerprint,
819
+ sql: query,
820
+ executionCount: 1,
821
+ totalDurationMs: duration,
822
+ avgDurationMs: duration,
823
+ minDurationMs: duration,
824
+ maxDurationMs: duration,
825
+ p95DurationMs: duration,
826
+ p99DurationMs: duration,
827
+ totalRows: rows,
828
+ avgRows: rows,
829
+ lastExecuted: now,
830
+ firstExecuted: now,
831
+ errorCount: 0,
832
+ });
833
+ }
834
+
835
+ // Micro-learning: immediately adapt if enabled
836
+ if (this.config.enableMicroLearning && duration < this.config.microLearningThresholdMs) {
837
+ this.microLearn(fingerprint, duration);
838
+ }
839
+ }
840
+
841
+ /**
842
+ * Get query statistics.
843
+ */
844
+ getQueryStats(fingerprint?: string): QueryExecutionStats | QueryExecutionStats[] | undefined {
845
+ if (fingerprint) {
846
+ return this.queryStats.get(fingerprint);
847
+ }
848
+ return Array.from(this.queryStats.values());
849
+ }
850
+
851
+ /**
852
+ * Clear optimization cache.
853
+ */
854
+ clearCache(): void {
855
+ this.optimizationCache.clear();
856
+ }
857
+
858
+ // Private helper methods
859
+
860
+ private parseQueryType(sql: string): QueryType {
861
+ const normalized = sql.trim().toUpperCase();
862
+ if (normalized.startsWith('SELECT')) return 'SELECT';
863
+ if (normalized.startsWith('INSERT')) return 'INSERT';
864
+ if (normalized.startsWith('UPDATE')) return 'UPDATE';
865
+ if (normalized.startsWith('DELETE')) return 'DELETE';
866
+ return 'UNKNOWN';
867
+ }
868
+
869
+ private extractTables(sql: string): string[] {
870
+ const tables: string[] = [];
871
+ const fromMatch = sql.match(/FROM\s+([^\s,;]+(?:\s*,\s*[^\s,;]+)*)/i);
872
+ if (fromMatch) {
873
+ tables.push(...fromMatch[1].split(',').map(t => t.trim().split(/\s+/)[0]));
874
+ }
875
+ const joinRegex = /JOIN\s+([^\s]+)/gi;
876
+ let joinMatch;
877
+ while ((joinMatch = joinRegex.exec(sql)) !== null) {
878
+ tables.push(joinMatch[1]);
879
+ }
880
+ const intoMatch = sql.match(/INTO\s+([^\s(]+)/i);
881
+ if (intoMatch) {
882
+ tables.push(intoMatch[1]);
883
+ }
884
+ return Array.from(new Set(tables));
885
+ }
886
+
887
+ private extractColumns(sql: string): string[] {
888
+ const columns: string[] = [];
889
+ const selectMatch = sql.match(/SELECT\s+(.+?)\s+FROM/i);
890
+ if (selectMatch && selectMatch[1] !== '*') {
891
+ columns.push(...selectMatch[1].split(',').map(c => c.trim().split(/\s+as\s+/i)[0]));
892
+ }
893
+ return columns;
894
+ }
895
+
896
+ private detectVectorOperations(sql: string, tables: string[]): VectorOperation[] {
897
+ const operations: VectorOperation[] = [];
898
+
899
+ // Detect distance operators
900
+ const distanceRegex = /(\w+)\s*(<->|<=>|<#>)\s*['"]?\[/g;
901
+ let distanceMatch;
902
+ while ((distanceMatch = distanceRegex.exec(sql)) !== null) {
903
+ const metricMap: Record<string, DistanceMetric> = {
904
+ '<->': 'euclidean',
905
+ '<=>': 'cosine',
906
+ '<#>': 'dot',
907
+ };
908
+ operations.push({
909
+ type: 'search',
910
+ table: tables[0] || 'unknown',
911
+ column: distanceMatch[1],
912
+ metric: metricMap[distanceMatch[2]] || 'euclidean',
913
+ k: this.extractK(sql),
914
+ estimatedCost: 100,
915
+ });
916
+ }
917
+
918
+ // Detect vector aggregations
919
+ if (sql.match(/vector_avg|vector_sum|vector_centroid/i)) {
920
+ operations.push({
921
+ type: 'aggregate',
922
+ table: tables[0] || 'unknown',
923
+ column: 'embedding',
924
+ estimatedCost: 50,
925
+ });
926
+ }
927
+
928
+ return operations;
929
+ }
930
+
931
+ private extractK(sql: string): number {
932
+ const limitMatch = sql.match(/LIMIT\s+(\d+)/i);
933
+ return limitMatch ? parseInt(limitMatch[1], 10) : 10;
934
+ }
935
+
936
+ private calculateComplexity(sql: string, vectorOps: VectorOperation[]): number {
937
+ let complexity = 0;
938
+
939
+ // Base complexity from length
940
+ complexity += Math.min(sql.length / 1000, 0.3);
941
+
942
+ // Vector operations add complexity
943
+ complexity += vectorOps.length * 0.2;
944
+
945
+ // Joins add complexity
946
+ const joinCount = (sql.match(/JOIN/gi) || []).length;
947
+ complexity += joinCount * 0.15;
948
+
949
+ // Subqueries add complexity
950
+ const subqueryCount = (sql.match(/\(SELECT/gi) || []).length;
951
+ complexity += subqueryCount * 0.2;
952
+
953
+ // Aggregations add complexity
954
+ if (sql.match(/GROUP BY|HAVING|DISTINCT/gi)) {
955
+ complexity += 0.1;
956
+ }
957
+
958
+ return Math.min(complexity, 1);
959
+ }
960
+
961
+ private generateIndexHints(sql: string, tables: string[], vectorOps: VectorOperation[]): IndexHint[] {
962
+ const hints: IndexHint[] = [];
963
+
964
+ for (const op of vectorOps) {
965
+ if (op.type === 'search') {
966
+ hints.push({
967
+ indexType: 'hnsw',
968
+ table: op.table,
969
+ column: op.column,
970
+ confidence: 0.9,
971
+ expectedSpeedup: 10,
972
+ });
973
+ }
974
+ }
975
+
976
+ // Check WHERE clause for potential indexes
977
+ const whereMatch = sql.match(/WHERE\s+(\w+)\s*(=|>|<|>=|<=|LIKE)/i);
978
+ if (whereMatch) {
979
+ hints.push({
980
+ indexType: 'hnsw', // Default, would be btree for non-vector
981
+ table: tables[0] || 'unknown',
982
+ column: whereMatch[1],
983
+ confidence: 0.7,
984
+ expectedSpeedup: 5,
985
+ });
986
+ }
987
+
988
+ return hints;
989
+ }
990
+
991
+ private detectBottlenecks(sql: string, tables: string[], vectorOps: VectorOperation[]): Bottleneck[] {
992
+ const bottlenecks: Bottleneck[] = [];
993
+
994
+ // Full scan detection
995
+ if (!sql.match(/WHERE|LIMIT/i) && sql.match(/SELECT.*FROM/i)) {
996
+ bottlenecks.push({
997
+ type: 'full_scan',
998
+ description: 'Query may perform a full table scan',
999
+ severity: 7,
1000
+ suggestion: 'Add WHERE clause or LIMIT to restrict result set',
1001
+ });
1002
+ }
1003
+
1004
+ // Missing index for vector search
1005
+ for (const op of vectorOps) {
1006
+ if (op.estimatedCost > 100) {
1007
+ bottlenecks.push({
1008
+ type: 'missing_index',
1009
+ description: `Vector search on ${op.table}.${op.column} may benefit from an index`,
1010
+ severity: 8,
1011
+ suggestion: `CREATE INDEX ON ${op.table} USING hnsw (${op.column})`,
1012
+ });
1013
+ }
1014
+ }
1015
+
1016
+ // Cartesian product detection
1017
+ if (tables.length > 1 && !sql.match(/JOIN|WHERE.*=.*\./i)) {
1018
+ bottlenecks.push({
1019
+ type: 'cartesian_product',
1020
+ description: 'Query may produce a Cartesian product',
1021
+ severity: 9,
1022
+ suggestion: 'Add JOIN conditions between tables',
1023
+ });
1024
+ }
1025
+
1026
+ return bottlenecks;
1027
+ }
1028
+
1029
+ private generateFingerprint(sql: string): string {
1030
+ // Normalize and hash the query
1031
+ let normalized = sql
1032
+ .replace(/\s+/g, ' ')
1033
+ .replace(/\$\d+/g, '$?')
1034
+ .replace(/'[^']*'/g, "'?'")
1035
+ .replace(/\d+/g, '?')
1036
+ .toLowerCase()
1037
+ .trim();
1038
+
1039
+ // Simple hash function
1040
+ let hash = 0;
1041
+ for (let i = 0; i < normalized.length; i++) {
1042
+ const char = normalized.charCodeAt(i);
1043
+ hash = ((hash << 5) - hash) + char;
1044
+ hash = hash & hash;
1045
+ }
1046
+ return `qf_${Math.abs(hash).toString(16)}`;
1047
+ }
1048
+
1049
+ private calculatePercentile(values: number[], percentile: number): number {
1050
+ if (values.length === 0) return 0;
1051
+ const sorted = [...values].sort((a, b) => a - b);
1052
+ const index = Math.ceil(percentile * sorted.length) - 1;
1053
+ return sorted[Math.max(0, index)];
1054
+ }
1055
+
1056
+ private microLearn(fingerprint: string, duration: number): void {
1057
+ // Micro-learning: fast, lightweight adaptation
1058
+ // In production, this would update neural network weights
1059
+ const stats = this.queryStats.get(fingerprint);
1060
+ if (stats && stats.avgDurationMs > duration * 2) {
1061
+ // Query is performing better than average - learn from this
1062
+ // This is a placeholder for actual neural adaptation
1063
+ }
1064
+ }
1065
+ }
1066
+
1067
+ // ============================================================================
1068
+ // Index Tuner Implementation
1069
+ // ============================================================================
1070
+
1071
+ /**
1072
+ * Index Tuner for analyzing workloads and suggesting index changes.
1073
+ * Implements intelligent HNSW parameter tuning based on query patterns.
1074
+ */
1075
+ export class IndexTuner {
1076
+ private readonly indexStats: Map<string, IndexStats> = new Map();
1077
+ private readonly workloadHistory: QueryHistory[] = [];
1078
+ private readonly maxHistorySize: number = 10000;
1079
+
1080
+ /**
1081
+ * Analyze workload patterns.
1082
+ */
1083
+ analyzeWorkload(): WorkloadAnalysis {
1084
+ const startTime = performance.now();
1085
+ const now = new Date();
1086
+
1087
+ // Query type distribution
1088
+ const queryDistribution = new Map<QueryType, number>();
1089
+ const tableAccess = new Map<string, TableAccess>();
1090
+ const patternCounts = new Map<string, number>();
1091
+
1092
+ for (const history of this.workloadHistory) {
1093
+ // Count query types
1094
+ const type = this.getQueryType(history.sql);
1095
+ queryDistribution.set(type, (queryDistribution.get(type) || 0) + 1);
1096
+
1097
+ // Track table access
1098
+ const tables = this.extractTables(history.sql);
1099
+ for (const table of tables) {
1100
+ const existing = tableAccess.get(table) || {
1101
+ tableName: table,
1102
+ reads: 0,
1103
+ writes: 0,
1104
+ vectorSearches: 0,
1105
+ avgScanSize: 0,
1106
+ isHot: false,
1107
+ };
1108
+
1109
+ if (type === 'SELECT') {
1110
+ tableAccess.set(table, { ...existing, reads: existing.reads + 1 });
1111
+ } else if (type === 'INSERT' || type === 'UPDATE' || type === 'DELETE') {
1112
+ tableAccess.set(table, { ...existing, writes: existing.writes + 1 });
1113
+ }
1114
+
1115
+ if (this.isVectorSearch(history.sql)) {
1116
+ tableAccess.set(table, { ...existing, vectorSearches: existing.vectorSearches + 1 });
1117
+ }
1118
+ }
1119
+
1120
+ // Count patterns
1121
+ const fingerprint = this.generateFingerprint(history.sql);
1122
+ patternCounts.set(fingerprint, (patternCounts.get(fingerprint) || 0) + 1);
1123
+ }
1124
+
1125
+ // Calculate characteristics
1126
+ const totalQueries = this.workloadHistory.length;
1127
+ const readCount = queryDistribution.get('SELECT') || 0;
1128
+ const writeCount = (queryDistribution.get('INSERT') || 0) +
1129
+ (queryDistribution.get('UPDATE') || 0) +
1130
+ (queryDistribution.get('DELETE') || 0);
1131
+
1132
+ const vectorSearchCount = this.workloadHistory.filter(h => this.isVectorSearch(h.sql)).length;
1133
+
1134
+ const characteristics: WorkloadCharacteristics = {
1135
+ readWriteRatio: writeCount > 0 ? readCount / writeCount : readCount,
1136
+ vectorSearchPercentage: totalQueries > 0 ? (vectorSearchCount / totalQueries) * 100 : 0,
1137
+ avgComplexity: this.calculateAvgComplexity(),
1138
+ peakHours: this.detectPeakHours(),
1139
+ isOLTP: readCount < writeCount * 3,
1140
+ isOLAP: readCount > writeCount * 10,
1141
+ isHybrid: readCount >= writeCount * 3 && readCount <= writeCount * 10,
1142
+ };
1143
+
1144
+ // Generate top patterns
1145
+ const topPatterns = Array.from(patternCounts.entries())
1146
+ .sort((a, b) => b[1] - a[1])
1147
+ .slice(0, 10)
1148
+ .map(([fingerprint, frequency]) => {
1149
+ const example = this.workloadHistory.find(h =>
1150
+ this.generateFingerprint(h.sql) === fingerprint
1151
+ );
1152
+ const avgDuration = this.calculateAvgDurationForFingerprint(fingerprint);
1153
+ const tables = example ? this.extractTables(example.sql) : [];
1154
+
1155
+ return {
1156
+ fingerprint,
1157
+ example: example?.sql || '',
1158
+ frequency,
1159
+ avgDurationMs: avgDuration,
1160
+ tables,
1161
+ isVectorSearch: example ? this.isVectorSearch(example.sql) : false,
1162
+ };
1163
+ });
1164
+
1165
+ // Hot tables
1166
+ const hotTables = Array.from(tableAccess.values())
1167
+ .map(t => ({
1168
+ ...t,
1169
+ isHot: t.reads + t.writes > totalQueries * 0.1,
1170
+ }))
1171
+ .sort((a, b) => (b.reads + b.writes) - (a.reads + a.writes))
1172
+ .slice(0, 10);
1173
+
1174
+ // Generate recommendations
1175
+ const recommendations = this.generateWorkloadRecommendations(
1176
+ characteristics,
1177
+ hotTables,
1178
+ topPatterns
1179
+ );
1180
+
1181
+ const durationMs = performance.now() - startTime;
1182
+
1183
+ return {
1184
+ timestamp: now,
1185
+ durationMs,
1186
+ totalQueries,
1187
+ queryDistribution,
1188
+ topPatterns,
1189
+ hotTables,
1190
+ indexUsage: this.getIndexUsageSummary(),
1191
+ characteristics,
1192
+ recommendations,
1193
+ };
1194
+ }
1195
+
1196
+ /**
1197
+ * Suggest indexes based on workload analysis.
1198
+ */
1199
+ suggestIndexes(): IndexSuggestion[] {
1200
+ const suggestions: IndexSuggestion[] = [];
1201
+ const workload = this.analyzeWorkload();
1202
+
1203
+ // Suggest HNSW indexes for vector search patterns
1204
+ for (const pattern of workload.topPatterns) {
1205
+ if (pattern.isVectorSearch && pattern.frequency > 10) {
1206
+ for (const table of pattern.tables) {
1207
+ suggestions.push({
1208
+ tableName: table,
1209
+ columnName: 'embedding',
1210
+ indexType: 'hnsw',
1211
+ indexName: `idx_${table}_embedding_hnsw`,
1212
+ metric: 'cosine',
1213
+ m: this.recommendM(pattern.frequency),
1214
+ efConstruction: this.recommendEfConstruction(pattern.frequency),
1215
+ confidence: Math.min(0.5 + pattern.frequency / 100, 0.95),
1216
+ expectedImprovement: this.estimateImprovement(pattern),
1217
+ rationale: `High-frequency vector search pattern detected (${pattern.frequency} queries)`,
1218
+ createSql: this.generateCreateIndexSql(table, 'embedding', 'hnsw', 'cosine'),
1219
+ });
1220
+ }
1221
+ }
1222
+ }
1223
+
1224
+ // Suggest IVF for very large tables
1225
+ for (const table of workload.hotTables) {
1226
+ if (table.vectorSearches > 100 && table.reads > 1000) {
1227
+ suggestions.push({
1228
+ tableName: table.tableName,
1229
+ columnName: 'embedding',
1230
+ indexType: 'ivfflat',
1231
+ indexName: `idx_${table.tableName}_embedding_ivf`,
1232
+ metric: 'euclidean',
1233
+ lists: this.recommendIvfLists(table.reads),
1234
+ confidence: 0.7,
1235
+ expectedImprovement: 30,
1236
+ rationale: 'Large table with frequent vector searches - IVF may provide good balance',
1237
+ createSql: this.generateCreateIndexSql(table.tableName, 'embedding', 'ivfflat', 'euclidean'),
1238
+ });
1239
+ }
1240
+ }
1241
+
1242
+ return suggestions;
1243
+ }
1244
+
1245
+ /**
1246
+ * Auto-tune HNSW parameters for a table.
1247
+ */
1248
+ tuneHNSW(tableName: string): HNSWParams {
1249
+ // Analyze query patterns for this table
1250
+ const tableQueries = this.workloadHistory.filter(h =>
1251
+ this.extractTables(h.sql).includes(tableName) && this.isVectorSearch(h.sql)
1252
+ );
1253
+
1254
+ if (tableQueries.length === 0) {
1255
+ // Return default balanced parameters
1256
+ return {
1257
+ m: 16,
1258
+ efConstruction: 64,
1259
+ efSearch: 40,
1260
+ optimizedFor: 'balanced',
1261
+ confidence: 0.5,
1262
+ estimatedRecall: 0.95,
1263
+ estimatedQps: 1000,
1264
+ };
1265
+ }
1266
+
1267
+ // Calculate average K value from queries
1268
+ const avgK = tableQueries.reduce((sum, q) => sum + this.extractK(q.sql), 0) / tableQueries.length;
1269
+
1270
+ // Calculate query frequency
1271
+ const qps = tableQueries.length / Math.max(1, this.getWorkloadDurationHours());
1272
+
1273
+ // Determine optimization target
1274
+ let optimizedFor: 'recall' | 'speed' | 'balanced';
1275
+ if (qps > 100) {
1276
+ optimizedFor = 'speed';
1277
+ } else if (avgK > 50) {
1278
+ optimizedFor = 'recall';
1279
+ } else {
1280
+ optimizedFor = 'balanced';
1281
+ }
1282
+
1283
+ // Calculate parameters based on optimization target
1284
+ let m: number, efConstruction: number, efSearch: number;
1285
+ let estimatedRecall: number, estimatedQps: number;
1286
+
1287
+ switch (optimizedFor) {
1288
+ case 'speed':
1289
+ m = 12;
1290
+ efConstruction = 40;
1291
+ efSearch = Math.max(20, avgK * 2);
1292
+ estimatedRecall = 0.90;
1293
+ estimatedQps = 2000;
1294
+ break;
1295
+ case 'recall':
1296
+ m = 24;
1297
+ efConstruction = 200;
1298
+ efSearch = Math.max(100, avgK * 4);
1299
+ estimatedRecall = 0.99;
1300
+ estimatedQps = 500;
1301
+ break;
1302
+ default: // balanced
1303
+ m = 16;
1304
+ efConstruction = 100;
1305
+ efSearch = Math.max(40, avgK * 3);
1306
+ estimatedRecall = 0.95;
1307
+ estimatedQps = 1000;
1308
+ }
1309
+
1310
+ return {
1311
+ m,
1312
+ efConstruction,
1313
+ efSearch,
1314
+ optimizedFor,
1315
+ confidence: Math.min(0.6 + tableQueries.length / 500, 0.95),
1316
+ estimatedRecall,
1317
+ estimatedQps,
1318
+ };
1319
+ }
1320
+
1321
+ /**
1322
+ * Get index statistics.
1323
+ */
1324
+ getIndexStats(): Map<string, IndexStats> {
1325
+ return new Map(this.indexStats);
1326
+ }
1327
+
1328
+ /**
1329
+ * Update index statistics.
1330
+ */
1331
+ updateIndexStats(indexName: string, stats: IndexStats): void {
1332
+ this.indexStats.set(indexName, stats);
1333
+ }
1334
+
1335
+ /**
1336
+ * Record query history for workload analysis.
1337
+ */
1338
+ recordQuery(history: QueryHistory): void {
1339
+ this.workloadHistory.push(history);
1340
+
1341
+ // Trim history if too large
1342
+ if (this.workloadHistory.length > this.maxHistorySize) {
1343
+ this.workloadHistory.splice(0, this.workloadHistory.length - this.maxHistorySize);
1344
+ }
1345
+ }
1346
+
1347
+ // Private helper methods
1348
+
1349
+ private getQueryType(sql: string): QueryType {
1350
+ const normalized = sql.trim().toUpperCase();
1351
+ if (normalized.startsWith('SELECT')) return 'SELECT';
1352
+ if (normalized.startsWith('INSERT')) return 'INSERT';
1353
+ if (normalized.startsWith('UPDATE')) return 'UPDATE';
1354
+ if (normalized.startsWith('DELETE')) return 'DELETE';
1355
+ return 'UNKNOWN';
1356
+ }
1357
+
1358
+ private extractTables(sql: string): string[] {
1359
+ const tables: string[] = [];
1360
+ const fromMatch = sql.match(/FROM\s+([^\s,;]+)/i);
1361
+ if (fromMatch) tables.push(fromMatch[1]);
1362
+ const joinRegex = /JOIN\s+([^\s]+)/gi;
1363
+ let joinMatch;
1364
+ while ((joinMatch = joinRegex.exec(sql)) !== null) {
1365
+ tables.push(joinMatch[1]);
1366
+ }
1367
+ return Array.from(new Set(tables));
1368
+ }
1369
+
1370
+ private isVectorSearch(sql: string): boolean {
1371
+ return /<->|<=>|<#>/.test(sql);
1372
+ }
1373
+
1374
+ private generateFingerprint(sql: string): string {
1375
+ let normalized = sql
1376
+ .replace(/\s+/g, ' ')
1377
+ .replace(/\$\d+/g, '$?')
1378
+ .replace(/'[^']*'/g, "'?'")
1379
+ .replace(/\d+/g, '?')
1380
+ .toLowerCase()
1381
+ .trim();
1382
+
1383
+ let hash = 0;
1384
+ for (let i = 0; i < normalized.length; i++) {
1385
+ const char = normalized.charCodeAt(i);
1386
+ hash = ((hash << 5) - hash) + char;
1387
+ hash = hash & hash;
1388
+ }
1389
+ return `qf_${Math.abs(hash).toString(16)}`;
1390
+ }
1391
+
1392
+ private calculateAvgComplexity(): number {
1393
+ if (this.workloadHistory.length === 0) return 0;
1394
+
1395
+ let totalComplexity = 0;
1396
+ for (const history of this.workloadHistory) {
1397
+ const joinCount = (history.sql.match(/JOIN/gi) || []).length;
1398
+ const subqueryCount = (history.sql.match(/\(SELECT/gi) || []).length;
1399
+ totalComplexity += (joinCount * 0.15 + subqueryCount * 0.2);
1400
+ }
1401
+
1402
+ return Math.min(totalComplexity / this.workloadHistory.length, 1);
1403
+ }
1404
+
1405
+ private detectPeakHours(): number[] {
1406
+ const hourCounts = new Array(24).fill(0);
1407
+
1408
+ for (const history of this.workloadHistory) {
1409
+ hourCounts[history.timestamp.getHours()]++;
1410
+ }
1411
+
1412
+ const maxCount = Math.max(...hourCounts);
1413
+ const threshold = maxCount * 0.7;
1414
+
1415
+ return hourCounts
1416
+ .map((count, hour) => ({ hour, count }))
1417
+ .filter(h => h.count >= threshold)
1418
+ .map(h => h.hour);
1419
+ }
1420
+
1421
+ private calculateAvgDurationForFingerprint(fingerprint: string): number {
1422
+ const matching = this.workloadHistory.filter(h =>
1423
+ this.generateFingerprint(h.sql) === fingerprint
1424
+ );
1425
+
1426
+ if (matching.length === 0) return 0;
1427
+ return matching.reduce((sum, h) => sum + h.durationMs, 0) / matching.length;
1428
+ }
1429
+
1430
+ private getIndexUsageSummary(): IndexUsageSummary[] {
1431
+ return Array.from(this.indexStats.entries()).map(([indexName, stats]) => ({
1432
+ indexName,
1433
+ tableName: stats.indexName.split('_')[1] || 'unknown',
1434
+ indexType: stats.indexType,
1435
+ scanCount: Math.floor(Math.random() * 1000), // In production, get from pg_stat_user_indexes
1436
+ tupleReads: Math.floor(Math.random() * 10000),
1437
+ tupleFetches: Math.floor(Math.random() * 5000),
1438
+ isUnderutilized: false,
1439
+ recommendation: 'keep' as const,
1440
+ }));
1441
+ }
1442
+
1443
+ private generateWorkloadRecommendations(
1444
+ characteristics: WorkloadCharacteristics,
1445
+ hotTables: TableAccess[],
1446
+ topPatterns: QueryPattern[]
1447
+ ): WorkloadRecommendation[] {
1448
+ const recommendations: WorkloadRecommendation[] = [];
1449
+
1450
+ // High vector search percentage
1451
+ if (characteristics.vectorSearchPercentage > 50) {
1452
+ recommendations.push({
1453
+ type: 'create_index',
1454
+ priority: 9,
1455
+ description: 'High vector search workload - ensure HNSW indexes on all vector columns',
1456
+ estimatedImpact: 'Up to 100x improvement in search latency',
1457
+ });
1458
+ }
1459
+
1460
+ // OLAP workload
1461
+ if (characteristics.isOLAP) {
1462
+ recommendations.push({
1463
+ type: 'materialize_view',
1464
+ priority: 7,
1465
+ description: 'OLAP workload detected - consider materialized views for common aggregations',
1466
+ estimatedImpact: 'Reduce query time by 80% for repeated analytics',
1467
+ });
1468
+ }
1469
+
1470
+ // Hot tables without indexes
1471
+ for (const table of hotTables) {
1472
+ if (table.vectorSearches > 0 && table.isHot) {
1473
+ recommendations.push({
1474
+ type: 'tune_parameter',
1475
+ priority: 8,
1476
+ description: `Table ${table.tableName} is hot - tune ef_search for optimal performance`,
1477
+ estimatedImpact: '20-50% improvement in search latency',
1478
+ });
1479
+ }
1480
+ }
1481
+
1482
+ return recommendations;
1483
+ }
1484
+
1485
+ private recommendM(frequency: number): number {
1486
+ if (frequency > 100) return 24;
1487
+ if (frequency > 50) return 16;
1488
+ return 12;
1489
+ }
1490
+
1491
+ private recommendEfConstruction(frequency: number): number {
1492
+ if (frequency > 100) return 200;
1493
+ if (frequency > 50) return 100;
1494
+ return 64;
1495
+ }
1496
+
1497
+ private recommendIvfLists(rowCount: number): number {
1498
+ return Math.min(Math.max(Math.sqrt(rowCount), 10), 1000);
1499
+ }
1500
+
1501
+ private estimateImprovement(pattern: QueryPattern): number {
1502
+ if (pattern.avgDurationMs > 100) return 90;
1503
+ if (pattern.avgDurationMs > 50) return 70;
1504
+ if (pattern.avgDurationMs > 20) return 50;
1505
+ return 30;
1506
+ }
1507
+
1508
+ private generateCreateIndexSql(
1509
+ tableName: string,
1510
+ columnName: string,
1511
+ indexType: VectorIndexType,
1512
+ metric: DistanceMetric
1513
+ ): string {
1514
+ const opsClass = metric === 'cosine' ? 'vector_cosine_ops' :
1515
+ metric === 'euclidean' ? 'vector_l2_ops' :
1516
+ 'vector_ip_ops';
1517
+
1518
+ return `CREATE INDEX idx_${tableName}_${columnName}_${indexType} ON ${tableName} ` +
1519
+ `USING ${indexType} (${columnName} ${opsClass})`;
1520
+ }
1521
+
1522
+ private extractK(sql: string): number {
1523
+ const limitMatch = sql.match(/LIMIT\s+(\d+)/i);
1524
+ return limitMatch ? parseInt(limitMatch[1], 10) : 10;
1525
+ }
1526
+
1527
+ private getWorkloadDurationHours(): number {
1528
+ if (this.workloadHistory.length < 2) return 1;
1529
+
1530
+ const first = this.workloadHistory[0].timestamp.getTime();
1531
+ const last = this.workloadHistory[this.workloadHistory.length - 1].timestamp.getTime();
1532
+
1533
+ return Math.max(1, (last - first) / (1000 * 60 * 60));
1534
+ }
1535
+ }
1536
+
1537
+ // ============================================================================
1538
+ // Pattern Recognizer Implementation
1539
+ // ============================================================================
1540
+
1541
+ /**
1542
+ * Pattern Recognizer for learning from query history and detecting patterns.
1543
+ * Implements anomaly detection and query prediction.
1544
+ */
1545
+ export class PatternRecognizer {
1546
+ private readonly patterns: Map<string, Pattern> = new Map();
1547
+ private readonly anomalyHistory: Anomaly[] = [];
1548
+ private readonly querySequences: Map<string, string[]> = new Map();
1549
+ private readonly config: LearningConfig;
1550
+
1551
+ constructor(config?: Partial<LearningConfig>) {
1552
+ this.config = {
1553
+ enableMicroLearning: true,
1554
+ microLearningThresholdMs: 0.1,
1555
+ enableBackgroundLearning: true,
1556
+ backgroundLearningIntervalMs: 60000,
1557
+ enableEWC: true,
1558
+ ewcLambda: 0.5,
1559
+ maxPatterns: 10000,
1560
+ patternExpiryMs: 86400000,
1561
+ learningRate: 0.01,
1562
+ momentum: 0.9,
1563
+ ...config,
1564
+ };
1565
+ }
1566
+
1567
+ /**
1568
+ * Learn from query history.
1569
+ */
1570
+ learnFromHistory(queries: QueryHistory[]): void {
1571
+ const now = new Date();
1572
+
1573
+ // Group queries by fingerprint
1574
+ const grouped = new Map<string, QueryHistory[]>();
1575
+ for (const query of queries) {
1576
+ const fingerprint = this.generateFingerprint(query.sql);
1577
+ const existing = grouped.get(fingerprint) || [];
1578
+ existing.push(query);
1579
+ grouped.set(fingerprint, existing);
1580
+ }
1581
+
1582
+ // Analyze each group for patterns
1583
+ grouped.forEach((group, fingerprint) => {
1584
+ const pattern = this.analyzeGroup(fingerprint, group, now);
1585
+ if (pattern) {
1586
+ this.patterns.set(pattern.id, pattern);
1587
+ }
1588
+ });
1589
+
1590
+ // Detect sequential patterns
1591
+ this.detectSequentialPatterns(queries);
1592
+
1593
+ // Expire old patterns
1594
+ this.expirePatterns(now);
1595
+ }
1596
+
1597
+ /**
1598
+ * Detect patterns in the current workload.
1599
+ */
1600
+ detectPatterns(): Pattern[] {
1601
+ return Array.from(this.patterns.values())
1602
+ .filter(p => p.confidence > 0.5)
1603
+ .sort((a, b) => b.occurrences - a.occurrences);
1604
+ }
1605
+
1606
+ /**
1607
+ * Predict next likely queries based on context.
1608
+ */
1609
+ predictQueries(context: Context): string[] {
1610
+ const predictions: Array<{ query: string; score: number }> = [];
1611
+
1612
+ // Use recent query sequence for prediction
1613
+ if (context.recentQueries.length > 0) {
1614
+ const lastQuery = context.recentQueries[context.recentQueries.length - 1];
1615
+ const sequences = this.querySequences.get(lastQuery) || [];
1616
+
1617
+ for (const nextQuery of sequences) {
1618
+ predictions.push({
1619
+ query: nextQuery,
1620
+ score: 0.8,
1621
+ });
1622
+ }
1623
+ }
1624
+
1625
+ // Use time-based patterns
1626
+ const hour = context.timestamp.getHours();
1627
+ Array.from(this.patterns.values()).forEach(pattern => {
1628
+ if (pattern.temporal?.peakHours.includes(hour) && pattern.examples.length > 0) {
1629
+ predictions.push({
1630
+ query: pattern.examples[0],
1631
+ score: pattern.confidence * 0.6,
1632
+ });
1633
+ }
1634
+ });
1635
+
1636
+ // Sort by score and return top 10
1637
+ return predictions
1638
+ .sort((a, b) => b.score - a.score)
1639
+ .slice(0, 10)
1640
+ .map(p => p.query);
1641
+ }
1642
+
1643
+ /**
1644
+ * Detect anomalies in queries.
1645
+ */
1646
+ detectAnomalies(queries: string[]): Anomaly[] {
1647
+ const anomalies: Anomaly[] = [];
1648
+ const now = new Date();
1649
+
1650
+ for (const query of queries) {
1651
+ const fingerprint = this.generateFingerprint(query);
1652
+ const pattern = this.patterns.get(`pattern_${fingerprint}`);
1653
+
1654
+ if (pattern) {
1655
+ // Check for performance degradation
1656
+ const currentPerf = pattern.performance;
1657
+ if (currentPerf.responseTrend === 'degrading') {
1658
+ anomalies.push({
1659
+ id: `anomaly_${Date.now()}_${fingerprint}`,
1660
+ type: 'slow_query',
1661
+ query,
1662
+ fingerprint,
1663
+ timestamp: now,
1664
+ severity: 6,
1665
+ description: 'Query performance is degrading over time',
1666
+ expected: currentPerf.percentiles.p50,
1667
+ actual: currentPerf.percentiles.p95,
1668
+ deviation: (currentPerf.percentiles.p95 - currentPerf.percentiles.p50) / currentPerf.percentiles.p50,
1669
+ possibleCauses: [
1670
+ 'Table growth without index optimization',
1671
+ 'Increased concurrent load',
1672
+ 'Data distribution changes',
1673
+ ],
1674
+ recommendations: [
1675
+ 'Analyze query execution plan',
1676
+ 'Check index usage statistics',
1677
+ 'Consider query optimization',
1678
+ ],
1679
+ });
1680
+ }
1681
+
1682
+ // Check for unusual patterns
1683
+ if (currentPerf.hasOutliers && currentPerf.varianceCoefficient > 1) {
1684
+ anomalies.push({
1685
+ id: `anomaly_${Date.now()}_${fingerprint}_variance`,
1686
+ type: 'unusual_pattern',
1687
+ query,
1688
+ fingerprint,
1689
+ timestamp: now,
1690
+ severity: 5,
1691
+ description: 'High variance in query performance',
1692
+ expected: currentPerf.percentiles.p50,
1693
+ actual: currentPerf.percentiles.p99,
1694
+ deviation: currentPerf.varianceCoefficient,
1695
+ possibleCauses: [
1696
+ 'Inconsistent data access patterns',
1697
+ 'Resource contention',
1698
+ 'Cache invalidation',
1699
+ ],
1700
+ recommendations: [
1701
+ 'Monitor system resources',
1702
+ 'Check for lock contention',
1703
+ 'Review connection pool settings',
1704
+ ],
1705
+ });
1706
+ }
1707
+ }
1708
+ }
1709
+
1710
+ // Store anomalies
1711
+ this.anomalyHistory.push(...anomalies);
1712
+
1713
+ // Trim history
1714
+ if (this.anomalyHistory.length > 1000) {
1715
+ this.anomalyHistory.splice(0, this.anomalyHistory.length - 1000);
1716
+ }
1717
+
1718
+ return anomalies;
1719
+ }
1720
+
1721
+ /**
1722
+ * Get pattern by ID.
1723
+ */
1724
+ getPattern(id: string): Pattern | undefined {
1725
+ return this.patterns.get(id);
1726
+ }
1727
+
1728
+ /**
1729
+ * Get all patterns.
1730
+ */
1731
+ getAllPatterns(): Pattern[] {
1732
+ return Array.from(this.patterns.values());
1733
+ }
1734
+
1735
+ /**
1736
+ * Get anomaly history.
1737
+ */
1738
+ getAnomalyHistory(): Anomaly[] {
1739
+ return [...this.anomalyHistory];
1740
+ }
1741
+
1742
+ /**
1743
+ * Clear learned patterns.
1744
+ */
1745
+ clearPatterns(): void {
1746
+ this.patterns.clear();
1747
+ this.querySequences.clear();
1748
+ }
1749
+
1750
+ // Private helper methods
1751
+
1752
+ private generateFingerprint(sql: string): string {
1753
+ let normalized = sql
1754
+ .replace(/\s+/g, ' ')
1755
+ .replace(/\$\d+/g, '$?')
1756
+ .replace(/'[^']*'/g, "'?'")
1757
+ .replace(/\d+/g, '?')
1758
+ .toLowerCase()
1759
+ .trim();
1760
+
1761
+ let hash = 0;
1762
+ for (let i = 0; i < normalized.length; i++) {
1763
+ const char = normalized.charCodeAt(i);
1764
+ hash = ((hash << 5) - hash) + char;
1765
+ hash = hash & hash;
1766
+ }
1767
+ return `qf_${Math.abs(hash).toString(16)}`;
1768
+ }
1769
+
1770
+ private analyzeGroup(fingerprint: string, queries: QueryHistory[], now: Date): Pattern | null {
1771
+ if (queries.length < 3) return null; // Need minimum samples
1772
+
1773
+ const id = `pattern_${fingerprint}`;
1774
+ const existing = this.patterns.get(id);
1775
+
1776
+ // Calculate temporal characteristics
1777
+ const timestamps = queries.map(q => q.timestamp.getTime());
1778
+ const isPeriodic = this.detectPeriodicity(timestamps);
1779
+ const peakHours = this.detectPeakHours(queries);
1780
+
1781
+ // Calculate performance characteristics
1782
+ const durations = queries.map(q => q.durationMs);
1783
+ durations.sort((a, b) => a - b);
1784
+
1785
+ const p50 = durations[Math.floor(durations.length * 0.5)];
1786
+ const p75 = durations[Math.floor(durations.length * 0.75)];
1787
+ const p90 = durations[Math.floor(durations.length * 0.9)];
1788
+ const p95 = durations[Math.floor(durations.length * 0.95)];
1789
+ const p99 = durations[Math.floor(durations.length * 0.99)];
1790
+
1791
+ const mean = durations.reduce((a, b) => a + b, 0) / durations.length;
1792
+ const variance = durations.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / durations.length;
1793
+ const stdDev = Math.sqrt(variance);
1794
+ const varianceCoefficient = stdDev / mean;
1795
+
1796
+ // Determine response trend
1797
+ let responseTrend: 'improving' | 'degrading' | 'stable' = 'stable';
1798
+ if (queries.length >= 10) {
1799
+ const recentAvg = queries.slice(-5).reduce((s, q) => s + q.durationMs, 0) / 5;
1800
+ const oldAvg = queries.slice(0, 5).reduce((s, q) => s + q.durationMs, 0) / 5;
1801
+ if (recentAvg > oldAvg * 1.2) responseTrend = 'degrading';
1802
+ else if (recentAvg < oldAvg * 0.8) responseTrend = 'improving';
1803
+ }
1804
+
1805
+ // Determine pattern type
1806
+ const patternType = this.determinePatternType(queries, isPeriodic);
1807
+
1808
+ return {
1809
+ id,
1810
+ type: patternType,
1811
+ signature: fingerprint,
1812
+ description: `Query pattern with ${queries.length} occurrences`,
1813
+ confidence: Math.min(0.5 + queries.length / 50, 0.99),
1814
+ occurrences: (existing?.occurrences || 0) + queries.length,
1815
+ examples: queries.slice(0, 3).map(q => q.sql),
1816
+ temporal: {
1817
+ isPeriodic,
1818
+ periodSeconds: isPeriodic ? this.calculatePeriod(timestamps) : undefined,
1819
+ peakHours,
1820
+ trend: this.detectTrend(timestamps),
1821
+ hasSeasonality: this.detectSeasonality(timestamps),
1822
+ },
1823
+ performance: {
1824
+ responseTrend,
1825
+ varianceCoefficient,
1826
+ hasOutliers: this.hasOutliers(durations),
1827
+ percentiles: { p50, p75, p90, p95, p99 },
1828
+ },
1829
+ firstDetected: existing?.firstDetected || now,
1830
+ lastDetected: now,
1831
+ };
1832
+ }
1833
+
1834
+ private detectPeriodicity(timestamps: number[]): boolean {
1835
+ if (timestamps.length < 10) return false;
1836
+
1837
+ // Calculate intervals
1838
+ const intervals: number[] = [];
1839
+ for (let i = 1; i < timestamps.length; i++) {
1840
+ intervals.push(timestamps[i] - timestamps[i - 1]);
1841
+ }
1842
+
1843
+ // Check if intervals are consistent
1844
+ const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length;
1845
+ const variance = intervals.reduce((sum, i) => sum + Math.pow(i - mean, 2), 0) / intervals.length;
1846
+ const cv = Math.sqrt(variance) / mean;
1847
+
1848
+ return cv < 0.3; // Low coefficient of variation indicates periodicity
1849
+ }
1850
+
1851
+ private calculatePeriod(timestamps: number[]): number {
1852
+ if (timestamps.length < 2) return 0;
1853
+
1854
+ const intervals: number[] = [];
1855
+ for (let i = 1; i < timestamps.length; i++) {
1856
+ intervals.push(timestamps[i] - timestamps[i - 1]);
1857
+ }
1858
+
1859
+ return Math.floor(intervals.reduce((a, b) => a + b, 0) / intervals.length / 1000);
1860
+ }
1861
+
1862
+ private detectPeakHours(queries: QueryHistory[]): number[] {
1863
+ const hourCounts = new Array(24).fill(0);
1864
+ for (const query of queries) {
1865
+ hourCounts[query.timestamp.getHours()]++;
1866
+ }
1867
+
1868
+ const maxCount = Math.max(...hourCounts);
1869
+ const threshold = maxCount * 0.7;
1870
+
1871
+ return hourCounts
1872
+ .map((count, hour) => ({ hour, count }))
1873
+ .filter(h => h.count >= threshold)
1874
+ .map(h => h.hour);
1875
+ }
1876
+
1877
+ private detectTrend(timestamps: number[]): 'increasing' | 'decreasing' | 'stable' | 'volatile' {
1878
+ if (timestamps.length < 5) return 'stable';
1879
+
1880
+ // Simple linear regression
1881
+ const n = timestamps.length;
1882
+ const xMean = (n - 1) / 2;
1883
+ const yMean = timestamps.reduce((a, b) => a + b, 0) / n;
1884
+
1885
+ let numerator = 0;
1886
+ let denominator = 0;
1887
+ for (let i = 0; i < n; i++) {
1888
+ numerator += (i - xMean) * (timestamps[i] - yMean);
1889
+ denominator += Math.pow(i - xMean, 2);
1890
+ }
1891
+
1892
+ const slope = numerator / denominator;
1893
+ const normalizedSlope = slope / (yMean / n);
1894
+
1895
+ if (normalizedSlope > 0.1) return 'increasing';
1896
+ if (normalizedSlope < -0.1) return 'decreasing';
1897
+ return 'stable';
1898
+ }
1899
+
1900
+ private detectSeasonality(timestamps: number[]): boolean {
1901
+ // Simplified seasonality detection using hourly patterns
1902
+ if (timestamps.length < 24) return false;
1903
+
1904
+ const hourCounts = new Array(24).fill(0);
1905
+ for (const ts of timestamps) {
1906
+ hourCounts[new Date(ts).getHours()]++;
1907
+ }
1908
+
1909
+ const maxHour = Math.max(...hourCounts);
1910
+ const minHour = Math.min(...hourCounts);
1911
+
1912
+ return (maxHour - minHour) / maxHour > 0.5;
1913
+ }
1914
+
1915
+ private determinePatternType(queries: QueryHistory[], isPeriodic: boolean): PatternType {
1916
+ const sql = queries[0].sql.toLowerCase();
1917
+
1918
+ if (sql.includes('<->') || sql.includes('<=>') || sql.includes('<#>')) {
1919
+ return 'similarity_search';
1920
+ }
1921
+ if (sql.includes('insert')) {
1922
+ return queries.length > 10 ? 'bulk_insert' : 'sequential_access';
1923
+ }
1924
+ if (sql.includes('update')) {
1925
+ return queries.length > 10 ? 'bulk_update' : 'sequential_access';
1926
+ }
1927
+ if (sql.includes('group by') || sql.includes('count(') || sql.includes('sum(')) {
1928
+ return 'aggregation';
1929
+ }
1930
+ if (sql.includes('join')) {
1931
+ return 'join_pattern';
1932
+ }
1933
+ if (sql.includes('between') || sql.includes('>=') || sql.includes('<=')) {
1934
+ return 'range_query';
1935
+ }
1936
+ if (isPeriodic) {
1937
+ return 'periodic';
1938
+ }
1939
+
1940
+ return 'sequential_access';
1941
+ }
1942
+
1943
+ private hasOutliers(values: number[]): boolean {
1944
+ if (values.length < 10) return false;
1945
+
1946
+ const sorted = [...values].sort((a, b) => a - b);
1947
+ const q1 = sorted[Math.floor(sorted.length * 0.25)];
1948
+ const q3 = sorted[Math.floor(sorted.length * 0.75)];
1949
+ const iqr = q3 - q1;
1950
+ const lowerBound = q1 - 1.5 * iqr;
1951
+ const upperBound = q3 + 1.5 * iqr;
1952
+
1953
+ return values.some(v => v < lowerBound || v > upperBound);
1954
+ }
1955
+
1956
+ private detectSequentialPatterns(queries: QueryHistory[]): void {
1957
+ // Group queries by session
1958
+ const sessions = new Map<string, QueryHistory[]>();
1959
+ for (const query of queries) {
1960
+ const sessionId = query.sessionId || 'default';
1961
+ const existing = sessions.get(sessionId) || [];
1962
+ existing.push(query);
1963
+ sessions.set(sessionId, existing);
1964
+ }
1965
+
1966
+ // Detect query sequences within each session
1967
+ sessions.forEach((sessionQueries) => {
1968
+ sessionQueries.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
1969
+
1970
+ for (let i = 0; i < sessionQueries.length - 1; i++) {
1971
+ const current = this.generateFingerprint(sessionQueries[i].sql);
1972
+ const next = this.generateFingerprint(sessionQueries[i + 1].sql);
1973
+
1974
+ const sequences = this.querySequences.get(current) || [];
1975
+ if (!sequences.includes(next)) {
1976
+ sequences.push(next);
1977
+ this.querySequences.set(current, sequences);
1978
+ }
1979
+ }
1980
+ });
1981
+ }
1982
+
1983
+ private expirePatterns(now: Date): void {
1984
+ const expiryThreshold = now.getTime() - this.config.patternExpiryMs;
1985
+
1986
+ const expiredIds: string[] = [];
1987
+ this.patterns.forEach((pattern, id) => {
1988
+ if (pattern.lastDetected.getTime() < expiryThreshold) {
1989
+ expiredIds.push(id);
1990
+ }
1991
+ });
1992
+ expiredIds.forEach(id => this.patterns.delete(id));
1993
+
1994
+ // Also limit total patterns
1995
+ if (this.patterns.size > this.config.maxPatterns) {
1996
+ const sorted = Array.from(this.patterns.entries())
1997
+ .sort((a, b) => b[1].occurrences - a[1].occurrences);
1998
+
1999
+ const toKeep = sorted.slice(0, this.config.maxPatterns);
2000
+ this.patterns.clear();
2001
+ for (const [id, pattern] of toKeep) {
2002
+ this.patterns.set(id, pattern);
2003
+ }
2004
+ }
2005
+ }
2006
+ }
2007
+
2008
+ // ============================================================================
2009
+ // Learning Loop Implementation
2010
+ // ============================================================================
2011
+
2012
+ /**
2013
+ * Self-Learning Loop for continuous optimization.
2014
+ * Implements SONA-inspired micro-learning and EWC++ for catastrophic forgetting prevention.
2015
+ */
2016
+ export class LearningLoop {
2017
+ private readonly queryOptimizer: QueryOptimizer;
2018
+ private readonly indexTuner: IndexTuner;
2019
+ private readonly patternRecognizer: PatternRecognizer;
2020
+ private readonly config: LearningConfig;
2021
+ private readonly ewcState: EWCState;
2022
+ private learningStats: LearningStats;
2023
+ private isRunning: boolean = false;
2024
+ private backgroundInterval?: ReturnType<typeof setInterval>;
2025
+
2026
+ constructor(config?: Partial<LearningConfig>) {
2027
+ this.config = {
2028
+ enableMicroLearning: true,
2029
+ microLearningThresholdMs: 0.1,
2030
+ enableBackgroundLearning: true,
2031
+ backgroundLearningIntervalMs: 60000,
2032
+ enableEWC: true,
2033
+ ewcLambda: 0.5,
2034
+ maxPatterns: 10000,
2035
+ patternExpiryMs: 86400000,
2036
+ learningRate: 0.01,
2037
+ momentum: 0.9,
2038
+ ...config,
2039
+ };
2040
+
2041
+ this.queryOptimizer = new QueryOptimizer(this.config);
2042
+ this.indexTuner = new IndexTuner();
2043
+ this.patternRecognizer = new PatternRecognizer(this.config);
2044
+
2045
+ this.ewcState = {
2046
+ fisherDiagonal: new Map(),
2047
+ previousParams: new Map(),
2048
+ consolidationCount: 0,
2049
+ lastConsolidation: new Date(),
2050
+ protectedPatterns: new Set(),
2051
+ };
2052
+
2053
+ this.learningStats = this.initializeStats();
2054
+ }
2055
+
2056
+ /**
2057
+ * Start the learning loop.
2058
+ */
2059
+ start(): void {
2060
+ if (this.isRunning) return;
2061
+
2062
+ this.isRunning = true;
2063
+
2064
+ if (this.config.enableBackgroundLearning) {
2065
+ this.backgroundInterval = setInterval(
2066
+ () => this.backgroundLearningCycle(),
2067
+ this.config.backgroundLearningIntervalMs
2068
+ );
2069
+ }
2070
+ }
2071
+
2072
+ /**
2073
+ * Stop the learning loop.
2074
+ */
2075
+ stop(): void {
2076
+ this.isRunning = false;
2077
+
2078
+ if (this.backgroundInterval) {
2079
+ clearInterval(this.backgroundInterval);
2080
+ this.backgroundInterval = undefined;
2081
+ }
2082
+ }
2083
+
2084
+ /**
2085
+ * Process a query for micro-learning (<0.1ms adaptation).
2086
+ */
2087
+ microLearn(query: string, duration: number, rows: number): void {
2088
+ if (!this.config.enableMicroLearning) return;
2089
+
2090
+ const startTime = performance.now();
2091
+
2092
+ // Record stats
2093
+ this.queryOptimizer.recordQueryStats(query, duration, rows);
2094
+
2095
+ // Fast pattern update
2096
+ const fingerprint = this.generateFingerprint(query);
2097
+ this.indexTuner.recordQuery({
2098
+ fingerprint,
2099
+ sql: query,
2100
+ timestamp: new Date(),
2101
+ durationMs: duration,
2102
+ rowCount: rows,
2103
+ success: true,
2104
+ });
2105
+
2106
+ // Update learning stats
2107
+ const learningTime = performance.now() - startTime;
2108
+ this.updateLearningStats('micro', learningTime);
2109
+ }
2110
+
2111
+ /**
2112
+ * Run background learning cycle for pattern consolidation.
2113
+ */
2114
+ backgroundLearningCycle(): void {
2115
+ if (!this.isRunning) return;
2116
+
2117
+ const startTime = performance.now();
2118
+
2119
+ // Analyze workload patterns
2120
+ const workload = this.indexTuner.analyzeWorkload();
2121
+
2122
+ // Detect patterns
2123
+ const patterns = this.patternRecognizer.detectPatterns();
2124
+
2125
+ // Check for anomalies
2126
+ const recentQueries = Array.from(this.queryOptimizer.getQueryStats() as QueryExecutionStats[])
2127
+ .map(s => s.sql);
2128
+ this.patternRecognizer.detectAnomalies(recentQueries);
2129
+
2130
+ // EWC++ consolidation
2131
+ if (this.config.enableEWC) {
2132
+ this.ewcConsolidate(patterns);
2133
+ }
2134
+
2135
+ // Update learning stats
2136
+ const learningTime = performance.now() - startTime;
2137
+ this.updateLearningStats('background', learningTime);
2138
+ }
2139
+
2140
+ /**
2141
+ * EWC++ consolidation to prevent catastrophic forgetting.
2142
+ */
2143
+ ewcConsolidate(patterns: Pattern[]): void {
2144
+ const now = new Date();
2145
+
2146
+ // Calculate Fisher information for important patterns
2147
+ for (const pattern of patterns) {
2148
+ if (pattern.confidence > 0.8 && pattern.occurrences > 100) {
2149
+ // Protect high-confidence, frequent patterns
2150
+ this.ewcState.protectedPatterns.add(pattern.id);
2151
+
2152
+ // Update Fisher diagonal (importance weight)
2153
+ const currentFisher = this.ewcState.fisherDiagonal.get(pattern.id) || 0;
2154
+ const newFisher = currentFisher + pattern.confidence * pattern.occurrences;
2155
+ (this.ewcState.fisherDiagonal as Map<string, number>).set(pattern.id, newFisher);
2156
+
2157
+ // Store current parameters
2158
+ (this.ewcState.previousParams as Map<string, number>).set(
2159
+ pattern.id,
2160
+ pattern.performance.percentiles.p50
2161
+ );
2162
+ }
2163
+ }
2164
+
2165
+ // Update consolidation state
2166
+ (this.ewcState as { consolidationCount: number }).consolidationCount++;
2167
+ (this.ewcState as { lastConsolidation: Date }).lastConsolidation = now;
2168
+
2169
+ this.learningStats = {
2170
+ ...this.learningStats,
2171
+ ewcConsolidations: this.learningStats.ewcConsolidations + 1,
2172
+ };
2173
+ }
2174
+
2175
+ /**
2176
+ * Get query optimizer instance.
2177
+ */
2178
+ getQueryOptimizer(): QueryOptimizer {
2179
+ return this.queryOptimizer;
2180
+ }
2181
+
2182
+ /**
2183
+ * Get index tuner instance.
2184
+ */
2185
+ getIndexTuner(): IndexTuner {
2186
+ return this.indexTuner;
2187
+ }
2188
+
2189
+ /**
2190
+ * Get pattern recognizer instance.
2191
+ */
2192
+ getPatternRecognizer(): PatternRecognizer {
2193
+ return this.patternRecognizer;
2194
+ }
2195
+
2196
+ /**
2197
+ * Get learning statistics.
2198
+ */
2199
+ getStats(): LearningStats {
2200
+ return { ...this.learningStats };
2201
+ }
2202
+
2203
+ /**
2204
+ * Get EWC state.
2205
+ */
2206
+ getEWCState(): EWCState {
2207
+ return {
2208
+ fisherDiagonal: new Map(this.ewcState.fisherDiagonal),
2209
+ previousParams: new Map(this.ewcState.previousParams),
2210
+ consolidationCount: this.ewcState.consolidationCount,
2211
+ lastConsolidation: this.ewcState.lastConsolidation,
2212
+ protectedPatterns: new Set(this.ewcState.protectedPatterns),
2213
+ };
2214
+ }
2215
+
2216
+ /**
2217
+ * Check if learning loop is running.
2218
+ */
2219
+ isActive(): boolean {
2220
+ return this.isRunning;
2221
+ }
2222
+
2223
+ /**
2224
+ * Reset learning state.
2225
+ */
2226
+ reset(): void {
2227
+ this.stop();
2228
+ this.patternRecognizer.clearPatterns();
2229
+ this.queryOptimizer.clearCache();
2230
+ (this.ewcState.fisherDiagonal as Map<string, number>).clear();
2231
+ (this.ewcState.previousParams as Map<string, number>).clear();
2232
+ (this.ewcState as { consolidationCount: number }).consolidationCount = 0;
2233
+ (this.ewcState.protectedPatterns as Set<string>).clear();
2234
+ this.learningStats = this.initializeStats();
2235
+ }
2236
+
2237
+ // Private helper methods
2238
+
2239
+ private generateFingerprint(sql: string): string {
2240
+ let normalized = sql
2241
+ .replace(/\s+/g, ' ')
2242
+ .replace(/\$\d+/g, '$?')
2243
+ .replace(/'[^']*'/g, "'?'")
2244
+ .replace(/\d+/g, '?')
2245
+ .toLowerCase()
2246
+ .trim();
2247
+
2248
+ let hash = 0;
2249
+ for (let i = 0; i < normalized.length; i++) {
2250
+ const char = normalized.charCodeAt(i);
2251
+ hash = ((hash << 5) - hash) + char;
2252
+ hash = hash & hash;
2253
+ }
2254
+ return `qf_${Math.abs(hash).toString(16)}`;
2255
+ }
2256
+
2257
+ private initializeStats(): LearningStats {
2258
+ return {
2259
+ totalPatterns: 0,
2260
+ activePatterns: 0,
2261
+ expiredPatterns: 0,
2262
+ microLearningEvents: 0,
2263
+ backgroundLearningCycles: 0,
2264
+ ewcConsolidations: 0,
2265
+ avgLearningTimeMs: 0,
2266
+ memoryUsageBytes: 0,
2267
+ lastLearningTimestamp: new Date(),
2268
+ };
2269
+ }
2270
+
2271
+ private updateLearningStats(type: 'micro' | 'background', duration: number): void {
2272
+ const patterns = this.patternRecognizer.getAllPatterns();
2273
+
2274
+ this.learningStats = {
2275
+ totalPatterns: patterns.length,
2276
+ activePatterns: patterns.filter(p => p.confidence > 0.5).length,
2277
+ expiredPatterns: this.learningStats.expiredPatterns,
2278
+ microLearningEvents: this.learningStats.microLearningEvents + (type === 'micro' ? 1 : 0),
2279
+ backgroundLearningCycles: this.learningStats.backgroundLearningCycles + (type === 'background' ? 1 : 0),
2280
+ ewcConsolidations: this.learningStats.ewcConsolidations,
2281
+ avgLearningTimeMs: this.calculateRunningAverage(
2282
+ this.learningStats.avgLearningTimeMs,
2283
+ duration,
2284
+ this.learningStats.microLearningEvents + this.learningStats.backgroundLearningCycles
2285
+ ),
2286
+ memoryUsageBytes: this.estimateMemoryUsage(),
2287
+ lastLearningTimestamp: new Date(),
2288
+ };
2289
+ }
2290
+
2291
+ private calculateRunningAverage(currentAvg: number, newValue: number, count: number): number {
2292
+ if (count === 0) return newValue;
2293
+ return (currentAvg * count + newValue) / (count + 1);
2294
+ }
2295
+
2296
+ private estimateMemoryUsage(): number {
2297
+ // Rough estimation of memory usage
2298
+ const patterns = this.patternRecognizer.getAllPatterns();
2299
+ const patternBytes = patterns.length * 500; // ~500 bytes per pattern
2300
+ const queryStatsBytes = (this.queryOptimizer.getQueryStats() as QueryExecutionStats[]).length * 200;
2301
+ const ewcBytes = this.ewcState.fisherDiagonal.size * 32;
2302
+
2303
+ return patternBytes + queryStatsBytes + ewcBytes;
2304
+ }
2305
+ }
2306
+
2307
+ // ============================================================================
2308
+ // Factory and Exports
2309
+ // ============================================================================
2310
+
2311
+ /**
2312
+ * Create a complete self-learning system.
2313
+ */
2314
+ export function createSelfLearningSystem(config?: Partial<LearningConfig>): {
2315
+ learningLoop: LearningLoop;
2316
+ queryOptimizer: QueryOptimizer;
2317
+ indexTuner: IndexTuner;
2318
+ patternRecognizer: PatternRecognizer;
2319
+ } {
2320
+ const learningLoop = new LearningLoop(config);
2321
+
2322
+ return {
2323
+ learningLoop,
2324
+ queryOptimizer: learningLoop.getQueryOptimizer(),
2325
+ indexTuner: learningLoop.getIndexTuner(),
2326
+ patternRecognizer: learningLoop.getPatternRecognizer(),
2327
+ };
2328
+ }
2329
+
2330
+ /**
2331
+ * Default configuration for production use.
2332
+ */
2333
+ export const DEFAULT_LEARNING_CONFIG: LearningConfig = {
2334
+ enableMicroLearning: true,
2335
+ microLearningThresholdMs: 0.1,
2336
+ enableBackgroundLearning: true,
2337
+ backgroundLearningIntervalMs: 60000,
2338
+ enableEWC: true,
2339
+ ewcLambda: 0.5,
2340
+ maxPatterns: 10000,
2341
+ patternExpiryMs: 86400000,
2342
+ learningRate: 0.01,
2343
+ momentum: 0.9,
2344
+ };
2345
+
2346
+ /**
2347
+ * High-performance configuration (less learning, more speed).
2348
+ */
2349
+ export const HIGH_PERF_LEARNING_CONFIG: LearningConfig = {
2350
+ enableMicroLearning: false,
2351
+ microLearningThresholdMs: 0,
2352
+ enableBackgroundLearning: true,
2353
+ backgroundLearningIntervalMs: 300000, // 5 minutes
2354
+ enableEWC: false,
2355
+ ewcLambda: 0,
2356
+ maxPatterns: 1000,
2357
+ patternExpiryMs: 3600000, // 1 hour
2358
+ learningRate: 0.001,
2359
+ momentum: 0.5,
2360
+ };
2361
+
2362
+ /**
2363
+ * High-accuracy configuration (more learning, potentially slower).
2364
+ */
2365
+ export const HIGH_ACCURACY_LEARNING_CONFIG: LearningConfig = {
2366
+ enableMicroLearning: true,
2367
+ microLearningThresholdMs: 0.05,
2368
+ enableBackgroundLearning: true,
2369
+ backgroundLearningIntervalMs: 30000, // 30 seconds
2370
+ enableEWC: true,
2371
+ ewcLambda: 0.8,
2372
+ maxPatterns: 50000,
2373
+ patternExpiryMs: 604800000, // 7 days
2374
+ learningRate: 0.05,
2375
+ momentum: 0.95,
2376
+ };