@unrdf/dark-matter 5.0.0

package/src/dark-matter/index-advisor.mjs

@@ -0,0 +1,242 @@
+/**
+ * @file Index Advisor - Recommend indexes based on query patterns
+ * @module @unrdf/dark-matter/index-advisor
+ */
+
+import { z } from 'zod';
+import { analyzeSparqlQuery } from './query-analyzer.mjs';
+
+/**
+ * @typedef {import('n3').Store} Store
+ */
+
+/**
+ * Index recommendation schema
+ */
+const IndexRecommendationSchema = z.object({
+  type: z.enum(['predicate', 'subject_predicate', 'object', 'composite']),
+  priority: z.enum(['low', 'medium', 'high', 'critical']),
+  estimatedBenefit: z.number().min(0).max(100),
+  reason: z.string(),
+  indexConfig: z.object({
+    fields: z.array(z.string()),
+    unique: z.boolean().optional(),
+  }),
+});
+
+/**
+ * Analyze index needs based on query log
+ * @param {Store} store - RDF store
+ * @param {Array<string>} queryLog - Array of executed queries
+ * @returns {Array<Object>} Index recommendations
+ *
+ * @throws {TypeError} If store or queryLog is invalid
+ *
+ * @example
+ * const recommendations = analyzeIndexNeeds(store, [query1, query2]);
+ * recommendations.forEach(r => console.log(r.type, r.priority));
+ */
+export function analyzeIndexNeeds(store, queryLog) {
+  if (!store || typeof store.getQuads !== 'function') {
+    throw new TypeError('analyzeIndexNeeds: store must be a valid Store instance');
+  }
+
+  if (!Array.isArray(queryLog)) {
+    throw new TypeError('analyzeIndexNeeds: queryLog must be an array');
+  }
+
+  const recommendations = [];
+  const predicateFrequency = new Map();
+  const subjectPredicateFrequency = new Map();
+
+  // Analyze query patterns
+  for (const query of queryLog) {
+    if (typeof query !== 'string') {
+      continue;
+    }
+
+    const analysis = analyzeSparqlQuery(query);
+
+    for (const pattern of analysis.patterns) {
+      // Track predicate frequency
+      if (!pattern.predicate.startsWith('?')) {
+        const count = predicateFrequency.get(pattern.predicate) || 0;
+        predicateFrequency.set(pattern.predicate, count + 1);
+      }
+
+      // Track subject+predicate combinations
+      if (!pattern.subject.startsWith('?') && !pattern.predicate.startsWith('?')) {
+        const key = `${pattern.subject}|${pattern.predicate}`;
+        const count = subjectPredicateFrequency.get(key) || 0;
+        subjectPredicateFrequency.set(key, count + 1);
+      }
+    }
+  }
+
+  // Generate predicate index recommendations
+  for (const [predicate, frequency] of predicateFrequency.entries()) {
+    if (frequency >= 3) {
+      const priority = frequency >= 10 ? 'high' : frequency >= 5 ? 'medium' : 'low';
+      const estimatedBenefit = Math.min(frequency * 10, 100);
+
+      recommendations.push({
+        type: 'predicate',
+        priority,
+        estimatedBenefit,
+        reason: `Predicate ${predicate} queried ${frequency} times`,
+        indexConfig: {
+          fields: ['predicate'],
+          unique: false,
+        },
+      });
+    }
+  }
+
+  // Generate composite index recommendations
+  for (const [_key, frequency] of subjectPredicateFrequency.entries()) {
+    if (frequency >= 2) {
+      const priority = frequency >= 5 ? 'high' : 'medium';
+      const estimatedBenefit = Math.min(frequency * 15, 100);
+
+      recommendations.push({
+        type: 'subject_predicate',
+        priority,
+        estimatedBenefit,
+        reason: `Subject+Predicate combination queried ${frequency} times`,
+        indexConfig: {
+          fields: ['subject', 'predicate'],
+          unique: false,
+        },
+      });
+    }
+  }
+
+  // Sort by estimated benefit
+  recommendations.sort((a, b) => b.estimatedBenefit - a.estimatedBenefit);
+
+  return recommendations.map(r => IndexRecommendationSchema.parse(r));
+}
+
+/**
+ * Suggest index for specific pattern
+ * @param {Object} pattern - Triple pattern
+ * @returns {Object} Index suggestion
+ *
+ * @throws {TypeError} If pattern is invalid
+ *
+ * @example
+ * const suggestion = suggestIndexForPattern({
+ *   subject: '?s',
+ *   predicate: '<http://xmlns.com/foaf/0.1/name>',
+ *   object: '?name'
+ * });
+ */
+export function suggestIndexForPattern(pattern) {
+  if (!pattern || typeof pattern !== 'object') {
+    throw new TypeError('suggestIndexForPattern: pattern must be an object');
+  }
+
+  const { subject, predicate, object } = pattern;
+
+  if (!subject || !predicate || !object) {
+    throw new TypeError('suggestIndexForPattern: pattern must have subject, predicate, and object');
+  }
+
+  // Specific predicate - recommend predicate index
+  if (!predicate.startsWith('?')) {
+    return {
+      type: 'predicate',
+      priority: 'high',
+      estimatedBenefit: 70,
+      reason: 'Specific predicate benefits from dedicated index',
+      indexConfig: {
+        fields: ['predicate'],
+      },
+    };
+  }
+
+  // Specific subject - recommend subject index
+  if (!subject.startsWith('?')) {
+    return {
+      type: 'subject_predicate',
+      priority: 'medium',
+      estimatedBenefit: 50,
+      reason: 'Specific subject can use subject-based index',
+      indexConfig: {
+        fields: ['subject'],
+      },
+    };
+  }
+
+  // Specific object - recommend object index
+  if (!object.startsWith('?')) {
+    return {
+      type: 'object',
+      priority: 'low',
+      estimatedBenefit: 30,
+      reason: 'Specific object may benefit from object index',
+      indexConfig: {
+        fields: ['object'],
+      },
+    };
+  }
+
+  // All wildcards - no specific index recommended
+  return {
+    type: 'composite',
+    priority: 'low',
+    estimatedBenefit: 10,
+    reason: 'Pattern too general for specific index',
+    indexConfig: {
+      fields: ['subject', 'predicate', 'object'],
+    },
+  };
+}
+
+/**
+ * Calculate index benefit for pattern
+ * @param {Object} pattern - Triple pattern
+ * @param {Object} indexConfig - Index configuration
+ * @returns {number} Benefit score 0-100
+ *
+ * @throws {TypeError} If parameters are invalid
+ *
+ * @example
+ * const benefit = calculateIndexBenefit(pattern, {
+ *   fields: ['predicate'],
+ *   unique: false
+ * });
+ */
+export function calculateIndexBenefit(pattern, indexConfig) {
+  if (!pattern || typeof pattern !== 'object') {
+    throw new TypeError('calculateIndexBenefit: pattern must be an object');
+  }
+
+  if (!indexConfig || typeof indexConfig !== 'object') {
+    throw new TypeError('calculateIndexBenefit: indexConfig must be an object');
+  }
+
+  const { subject, predicate, object } = pattern;
+  const { fields } = indexConfig;
+
+  if (!Array.isArray(fields)) {
+    throw new TypeError('calculateIndexBenefit: indexConfig.fields must be an array');
+  }
+
+  let benefit = 0;
+
+  // Check if indexed fields are bound (not variables)
+  for (const field of fields) {
+    if (field === 'subject' && !subject.startsWith('?')) {
+      benefit += 30;
+    }
+    if (field === 'predicate' && !predicate.startsWith('?')) {
+      benefit += 40;
+    }
+    if (field === 'object' && !object.startsWith('?')) {
+      benefit += 30;
+    }
+  }
+
+  return Math.min(benefit, 100);
+}

package/src/dark-matter/index.mjs

@@ -0,0 +1,244 @@
+/**
+ * @file Dark Matter 80/20 Query Optimization - Main Export
+ * @module dark-matter
+ *
+ * @description
+ * Main entry point for Dark Matter 80/20 query optimization system.
+ * Provides integrated query analysis, critical path identification,
+ * and query optimization following the 80/20 principle.
+ */
+
+import { QueryAnalyzer, createQueryAnalyzer } from './query-analyzer.mjs';
+import { CriticalPathIdentifier, createCriticalPathIdentifier } from './critical-path.mjs';
+import { DarkMatterOptimizer, createDarkMatterOptimizer } from './optimizer.mjs';
+
+/**
+ * Integrated Dark Matter query optimization system
+ */
+export class DarkMatterQuerySystem {
+  /**
+   * Create a new Dark Matter query system
+   * @param {Object} [config] - Configuration
+   */
+  constructor(config = {}) {
+    this.analyzer = createQueryAnalyzer(config.analyzer);
+    this.criticalPath = createCriticalPathIdentifier(config.criticalPath);
+    this.optimizer = createDarkMatterOptimizer(config.optimizer);
+
+    this.config = {
+      enableAutoOptimization: config.enableAutoOptimization !== false,
+      complexityThreshold: config.complexityThreshold || 100,
+      ...config,
+    };
+  }
+
+  /**
+   * Analyze a query
+   * @param {string} query - SPARQL query
+   * @param {string} [queryId] - Optional query identifier
+   * @returns {Object} Analysis result
+   */
+  analyze(query, queryId = null) {
+    return this.analyzer.analyze(query, queryId);
+  }
+
+  /**
+   * Log query execution for critical path analysis
+   * @param {string} queryId - Query identifier
+   * @param {string} query - SPARQL query
+   * @param {number} executionTime - Execution time in ms
+   * @param {Object} [metadata] - Optional metadata
+   */
+  logExecution(queryId, query, executionTime, metadata = {}) {
+    this.criticalPath.logExecution(queryId, query, executionTime, metadata);
+  }
+
+  /**
+   * Identify critical queries
+   * @returns {Object} Critical path analysis
+   */
+  identifyCriticalQueries() {
+    return this.criticalPath.identify();
+  }
+
+  /**
+   * Optimize a query
+   * @param {string} query - SPARQL query
+   * @param {Object} [analysis] - Optional pre-computed analysis
+   * @returns {Object} Optimization result
+   */
+  optimize(query, analysis = null) {
+    // Analyze first if not provided
+    if (!analysis) {
+      analysis = this.analyzer.analyze(query);
+    }
+
+    // Only optimize if above complexity threshold
+    if (analysis.complexity.score < this.config.complexityThreshold) {
+      return {
+        original: query,
+        optimized: query,
+        rules: [],
+        estimatedImprovement: {
+          before: analysis.complexity.score,
+          after: analysis.complexity.score,
+          percentageGain: 0,
+        },
+        timestamp: Date.now(),
+        skipped: true,
+        reason: 'Query complexity below threshold',
+      };
+    }
+
+    return this.optimizer.optimize(query, analysis);
+  }
+
+  /**
+   * Analyze and optimize a query in one step
+   * @param {string} query - SPARQL query
+   * @param {string} [queryId] - Optional query identifier
+   * @returns {Object} Combined analysis and optimization
+   */
+  analyzeAndOptimize(query, queryId = null) {
+    const analysis = this.analyze(query, queryId);
+    const optimization = this.optimize(query, analysis);
+
+    return {
+      analysis,
+      optimization,
+      shouldOptimize: !optimization.skipped,
+    };
+  }
+
+  /**
+   * Process query execution: analyze, log, and optionally optimize
+   * @param {string} query - SPARQL query
+   * @param {number} executionTime - Execution time in ms
+   * @param {string} [queryId] - Optional query identifier
+   * @returns {Object} Processing result
+   */
+  processExecution(query, executionTime, queryId = null) {
+    const analysis = this.analyze(query, queryId);
+    const id = queryId || analysis.queryId;
+
+    // Log execution
+    this.logExecution(id, query, executionTime, {
+      complexity: analysis.complexity.score,
+      expensiveOps: analysis.expensiveOperations.length,
+    });
+
+    // Auto-optimize if enabled and above threshold
+    let optimization = null;
+    if (this.config.enableAutoOptimization) {
+      optimization = this.optimize(query, analysis);
+    }
+
+    return {
+      queryId: id,
+      analysis,
+      optimization,
+      logged: true,
+    };
+  }
+
+  /**
+   * Get comprehensive statistics
+   * @returns {Object} Statistics
+   */
+  getStats() {
+    let criticalPathMetrics = null;
+
+    try {
+      criticalPathMetrics = this.criticalPath.identify().metrics;
+    } catch (error) {
+      // Not enough data yet for critical path analysis
+      criticalPathMetrics = {
+        error: error.message,
+        totalQueries: 0,
+        criticalQueryCount: 0,
+        criticalQueryPercentage: 0,
+        totalExecutionTime: 0,
+        criticalExecutionTime: 0,
+        impactRatio: 0,
+        avgExecutionTime: 0,
+        p50: 0,
+        p90: 0,
+        p99: 0,
+      };
+    }
+
+    return {
+      analyzer: this.analyzer.getStats(),
+      criticalPath: criticalPathMetrics,
+      optimizer: this.optimizer.getStats(),
+    };
+  }
+
+  /**
+   * Generate full report
+   * @returns {string} Markdown report
+   */
+  getReport() {
+    let report = '# Dark Matter 80/20 Query Optimization Report\n\n';
+
+    // Analyzer stats
+    const analyzerStats = this.analyzer.getStats();
+    report += '## Query Analysis\n\n';
+    report += `- **Total Queries Analyzed**: ${analyzerStats.totalAnalyzed}\n`;
+    report += `- **Complex Queries**: ${analyzerStats.complexQueries}\n`;
+    report += `- **Simple Queries**: ${analyzerStats.simpleQueries}\n`;
+    report += `- **Complexity Ratio**: ${(analyzerStats.complexQueryRatio * 100).toFixed(1)}%\n`;
+    report += `- **Average Complexity**: ${analyzerStats.avgComplexity.toFixed(2)}\n\n`;
+
+    // Critical path
+    try {
+      const criticalPathReport = this.criticalPath.getReport();
+      report += criticalPathReport + '\n\n';
+    } catch (error) {
+      report += '## Critical Path Analysis\n\n';
+      report += `*Insufficient data for analysis: ${error.message}*\n\n`;
+    }
+
+    // Optimizer stats
+    const optimizerStats = this.optimizer.getStats();
+    report += '## Optimization Statistics\n\n';
+    report += `- **Total Optimizations**: ${optimizerStats.totalOptimizations}\n`;
+    report += '- **Rules Applied**:\n';
+
+    for (const [rule, count] of Object.entries(optimizerStats.rulesApplied)) {
+      report += `  - ${rule}: ${count}\n`;
+    }
+
+    return report;
+  }
+
+  /**
+   * Clear all data
+   */
+  clear() {
+    this.analyzer.resetStats();
+    this.criticalPath.clearLogs();
+    this.optimizer.resetStats();
+  }
+}
+
+/**
+ * Create a Dark Matter query system
+ * @param {Object} [config] - Configuration
+ * @returns {DarkMatterQuerySystem} Query system
+ */
+export function createDarkMatterQuerySystem(config = {}) {
+  return new DarkMatterQuerySystem(config);
+}
+
+// Re-export individual components
+export {
+  QueryAnalyzer,
+  createQueryAnalyzer,
+  CriticalPathIdentifier,
+  createCriticalPathIdentifier,
+  DarkMatterOptimizer,
+  createDarkMatterOptimizer,
+};
+
+export default DarkMatterQuerySystem;