@unrdf/dark-matter 5.0.0

package/src/dark-matter/query-analyzer.mjs

@@ -0,0 +1,442 @@
+/**
+ * @file Dark Matter 80/20 Query Analyzer
+ * @module dark-matter/query-analyzer
+ *
+ * @description
+ * Analyzes SPARQL queries to extract patterns, calculate complexity scores,
+ * and identify expensive operations for 80/20 optimization.
+ */
+
+import { analyzeSPARQLQuery, extractVariables } from '../../utils/sparql-utils.mjs';
+import { z } from 'zod';
+
+/**
+ * Schema for query analysis result
+ */
+const QueryAnalysisSchema = z.object({
+  queryId: z.string(),
+  query: z.string(),
+  type: z.enum(['SELECT', 'ASK', 'CONSTRUCT', 'DESCRIBE', 'UNKNOWN']),
+  patterns: z.array(
+    z.object({
+      type: z.string(),
+      subject: z.string().optional(),
+      predicate: z.string().optional(),
+      object: z.string().optional(),
+      complexity: z.number(),
+    })
+  ),
+  filters: z.array(z.string()),
+  joins: z.array(
+    z.object({
+      type: z.string(),
+      variables: z.array(z.string()),
+      estimatedCost: z.number(),
+    })
+  ),
+  aggregations: z.array(z.string()),
+  complexity: z.object({
+    score: z.number(),
+    patternCount: z.number(),
+    filterCount: z.number(),
+    joinCount: z.number(),
+    aggregationCount: z.number(),
+    variableCount: z.number(),
+    estimatedRows: z.number(),
+  }),
+  expensiveOperations: z.array(
+    z.object({
+      type: z.string(),
+      cost: z.number(),
+      reason: z.string(),
+    })
+  ),
+  timestamp: z.number(),
+});
+
+/**
+ * Query Analyzer for Dark Matter 80/20 optimization
+ */
+export class QueryAnalyzer {
+  /**
+   * Create a new query analyzer
+   * @param {Object} [config] - Configuration
+   */
+  constructor(config = {}) {
+    this.config = {
+      complexityThreshold: config.complexityThreshold || 100,
+      expensiveOperationThreshold: config.expensiveOperationThreshold || 50,
+      joinCostMultiplier: config.joinCostMultiplier || 10,
+      filterCostMultiplier: config.filterCostMultiplier || 5,
+      aggregationCostMultiplier: config.aggregationCostMultiplier || 8,
+      ...config,
+    };
+
+    this.stats = {
+      totalAnalyzed: 0,
+      complexQueries: 0,
+      simpleQueries: 0,
+      avgComplexity: 0,
+    };
+  }
+
+  /**
+   * Analyze a SPARQL query
+   * @param {string} query - SPARQL query string
+   * @param {string} [queryId] - Optional query identifier
+   * @param {Object} [metadata] - Optional metadata
+   * @returns {Object} Query analysis result
+   */
+  analyze(query, queryId = null, metadata = {}) {
+    const analysis = analyzeSPARQLQuery(query);
+    const patterns = this._extractPatterns(query, analysis);
+    const filters = this._extractFilters(query, analysis);
+    const joins = this._identifyJoins(query, analysis);
+    const aggregations = this._extractAggregations(query);
+
+    // Calculate complexity score
+    const complexity = this._calculateComplexity(patterns, filters, joins, aggregations, analysis);
+
+    // Identify expensive operations
+    const expensiveOperations = this._identifyExpensiveOperations(
+      patterns,
+      filters,
+      joins,
+      aggregations,
+      complexity
+    );
+
+    const result = {
+      queryId: queryId || `query-${Date.now()}`,
+      query,
+      type: analysis.type,
+      patterns,
+      filters,
+      joins,
+      aggregations,
+      complexity,
+      expensiveOperations,
+      timestamp: Date.now(),
+      metadata,
+    };
+
+    // Update stats
+    this.stats.totalAnalyzed++;
+    if (complexity.score >= this.config.complexityThreshold) {
+      this.stats.complexQueries++;
+    } else {
+      this.stats.simpleQueries++;
+    }
+
+    // Update average complexity
+    this.stats.avgComplexity =
+      (this.stats.avgComplexity * (this.stats.totalAnalyzed - 1) + complexity.score) /
+      this.stats.totalAnalyzed;
+
+    return QueryAnalysisSchema.parse(result);
+  }
+
+  /**
+   * Extract triple patterns from query
+   * @param {string} query - SPARQL query
+   * @param {Object} analysis - Basic analysis
+   * @returns {Array} Triple patterns
+   * @private
+   */
+  _extractPatterns(query, _analysis) {
+    const patterns = [];
+
+    // Extract WHERE clause
+    const whereMatch = query.match(/WHERE\s*\{([^}]+)\}/is);
+    if (!whereMatch) return patterns;
+
+    const whereClause = whereMatch[1];
+
+    // Simple pattern extraction (s p o .)
+    const triplePattern = /(\??\w+|<[^>]+>)\s+(\??\w+|<[^>]+>)\s+(\??\w+|<[^>]+>|"[^"]*")\s*\./g;
+    let match;
+
+    while ((match = triplePattern.exec(whereClause)) !== null) {
+      const [, subject, predicate, object] = match;
+
+      // Calculate pattern complexity
+      let complexity = 1;
+
+      // Variable in subject position = join likely
+      if (subject.startsWith('?')) complexity += 5;
+
+      // Variable in predicate position = very expensive
+      if (predicate.startsWith('?')) complexity += 10;
+
+      // Variable in object position = filter likely
+      if (object.startsWith('?')) complexity += 3;
+
+      patterns.push({
+        type: 'triple',
+        subject,
+        predicate,
+        object,
+        complexity,
+      });
+    }
+
+    return patterns;
+  }
+
+  /**
+   * Extract FILTER clauses
+   * @param {string} query - SPARQL query
+   * @param {Object} analysis - Basic analysis
+   * @returns {Array} Filters
+   * @private
+   */
+  _extractFilters(query, _analysis) {
+    const filters = [];
+    const filterPattern = /FILTER\s*\(([^)]+)\)/gi;
+    let match;
+
+    while ((match = filterPattern.exec(query)) !== null) {
+      filters.push(match[1].trim());
+    }
+
+    return filters;
+  }
+
+  /**
+   * Identify JOIN operations
+   * @param {string} query - SPARQL query
+   * @param {Object} analysis - Basic analysis
+   * @returns {Array} Joins
+   * @private
+   */
+  _identifyJoins(query, _analysis) {
+    const joins = [];
+    const variables = extractVariables(query);
+
+    // Simple heuristic: if a variable appears multiple times, it's a join
+    const variableCounts = new Map();
+
+    for (const variable of variables) {
+      const regex = new RegExp(`\\?${variable}`, 'g');
+      const matches = query.match(regex);
+      variableCounts.set(variable, matches ? matches.length : 0);
+    }
+
+    // Identify joins based on shared variables
+    for (const [variable, count] of variableCounts.entries()) {
+      if (count >= 2) {
+        // Estimate join cost based on number of occurrences
+        const estimatedCost = count * this.config.joinCostMultiplier;
+
+        joins.push({
+          type: 'variable-join',
+          variables: [variable],
+          estimatedCost,
+        });
+      }
+    }
+
+    // Detect OPTIONAL joins (left outer joins)
+    if (query.includes('OPTIONAL')) {
+      joins.push({
+        type: 'optional-join',
+        variables: [],
+        estimatedCost: 20, // OPTIONAL is typically expensive
+      });
+    }
+
+    // Detect UNION (union joins)
+    if (query.includes('UNION')) {
+      joins.push({
+        type: 'union',
+        variables: [],
+        estimatedCost: 30, // UNION is very expensive
+      });
+    }
+
+    return joins;
+  }
+
+  /**
+   * Extract aggregation operations
+   * @param {string} query - SPARQL query
+   * @returns {Array} Aggregations
+   * @private
+   */
+  _extractAggregations(query) {
+    const aggregations = [];
+    const aggPattern = /(COUNT|SUM|AVG|MIN|MAX|GROUP_CONCAT|SAMPLE)\s*\(/gi;
+    let match;
+
+    while ((match = aggPattern.exec(query)) !== null) {
+      aggregations.push(match[1].toUpperCase());
+    }
+
+    return aggregations;
+  }
+
+  /**
+   * Calculate query complexity score
+   * @param {Array} patterns - Triple patterns
+   * @param {Array} filters - Filters
+   * @param {Array} joins - Joins
+   * @param {Array} aggregations - Aggregations
+   * @param {Object} analysis - Basic analysis
+   * @returns {Object} Complexity metrics
+   * @private
+   */
+  _calculateComplexity(patterns, filters, joins, aggregations, analysis) {
+    let score = 0;
+
+    // Base cost from patterns
+    const patternCost = patterns.reduce((sum, p) => sum + p.complexity, 0);
+    score += patternCost;
+
+    // Filter cost
+    const filterCost = filters.length * this.config.filterCostMultiplier;
+    score += filterCost;
+
+    // Join cost
+    const joinCost = joins.reduce((sum, j) => sum + j.estimatedCost, 0);
+    score += joinCost;
+
+    // Aggregation cost
+    const aggregationCost = aggregations.length * this.config.aggregationCostMultiplier;
+    score += aggregationCost;
+
+    // Complexity modifiers
+    if (analysis.hasGroupBy) score *= 1.5;
+    if (analysis.hasOrderBy) score *= 1.2;
+    if (analysis.hasDistinct) score *= 1.3;
+
+    // Estimate result rows (used for optimization decisions)
+    let estimatedRows = 100; // Base estimate
+
+    if (joins.length > 0) {
+      estimatedRows *= Math.pow(10, joins.length); // Cartesian product estimation
+    }
+
+    if (filters.length > 0) {
+      estimatedRows /= filters.length * 2; // Filters reduce results
+    }
+
+    return {
+      score: Math.round(score),
+      patternCount: patterns.length,
+      filterCount: filters.length,
+      joinCount: joins.length,
+      aggregationCount: aggregations.length,
+      variableCount: analysis.variables.length,
+      estimatedRows: Math.round(estimatedRows),
+    };
+  }
+
+  /**
+   * Identify expensive operations
+   * @param {Array} patterns - Triple patterns
+   * @param {Array} filters - Filters
+   * @param {Array} joins - Joins
+   * @param {Array} aggregations - Aggregations
+   * @param {Object} complexity - Complexity metrics
+   * @returns {Array} Expensive operations
+   * @private
+   */
+  _identifyExpensiveOperations(patterns, filters, joins, aggregations, complexity) {
+    const expensive = [];
+
+    // Check for expensive patterns
+    for (const pattern of patterns) {
+      if (pattern.complexity >= this.config.expensiveOperationThreshold / 5) {
+        expensive.push({
+          type: 'pattern',
+          cost: pattern.complexity,
+          reason: `High complexity pattern: ${pattern.subject} ${pattern.predicate} ${pattern.object}`,
+        });
+      }
+
+      // Variable predicates are very expensive
+      if (pattern.predicate.startsWith('?')) {
+        expensive.push({
+          type: 'variable-predicate',
+          cost: 100,
+          reason: `Variable predicate ${pattern.predicate} requires full graph scan`,
+        });
+      }
+    }
+
+    // Check for expensive joins
+    for (const join of joins) {
+      if (join.estimatedCost >= this.config.expensiveOperationThreshold) {
+        expensive.push({
+          type: 'join',
+          cost: join.estimatedCost,
+          reason: `${join.type} with cost ${join.estimatedCost}`,
+        });
+      }
+    }
+
+    // UNION is always expensive
+    if (joins.some(j => j.type === 'union')) {
+      expensive.push({
+        type: 'union',
+        cost: 50,
+        reason: 'UNION requires multiple query executions',
+      });
+    }
+
+    // Aggregations without GROUP BY are expensive on large datasets
+    if (aggregations.length > 0 && complexity.estimatedRows > 1000) {
+      expensive.push({
+        type: 'aggregation',
+        cost: aggregations.length * this.config.aggregationCostMultiplier,
+        reason: `${aggregations.length} aggregation(s) on ~${complexity.estimatedRows} rows`,
+      });
+    }
+
+    // Unfiltered queries with high estimated rows
+    if (filters.length === 0 && complexity.estimatedRows > 10000) {
+      expensive.push({
+        type: 'unfiltered',
+        cost: 75,
+        reason: `No filters with estimated ${complexity.estimatedRows} rows`,
+      });
+    }
+
+    return expensive.sort((a, b) => b.cost - a.cost);
+  }
+
+  /**
+   * Get analyzer statistics
+   * @returns {Object} Statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      complexQueryRatio:
+        this.stats.totalAnalyzed > 0 ? this.stats.complexQueries / this.stats.totalAnalyzed : 0,
+    };
+  }
+
+  /**
+   * Reset statistics
+   */
+  resetStats() {
+    this.stats = {
+      totalAnalyzed: 0,
+      complexQueries: 0,
+      simpleQueries: 0,
+      avgComplexity: 0,
+    };
+  }
+}
+
+/**
+ * Create a query analyzer instance
+ * @param {Object} [config] - Configuration
+ * @returns {QueryAnalyzer} Query analyzer
+ */
+export function createQueryAnalyzer(config = {}) {
+  return new QueryAnalyzer(config);
+}
+
+export default QueryAnalyzer;

package/src/dark-matter/query-optimizer.mjs

@@ -0,0 +1,283 @@
+/**
+ * @file SPARQL Query Optimizer - Reorder patterns for efficiency
+ * @module @unrdf/dark-matter/query-optimizer
+ */
+
+import { z } from 'zod';
+import { analyzeSparqlQuery } from './query-analyzer.mjs';
+
+/**
+ * @typedef {import('n3').Store} Store
+ */
+
+/**
+ * Optimization result schema
+ */
+const OptimizationResultSchema = z.object({
+  originalQuery: z.string(),
+  optimizedQuery: z.string(),
+  changes: z.array(
+    z.object({
+      type: z.string(),
+      description: z.string(),
+    })
+  ),
+  estimatedSpeedup: z.number(),
+});
+
+/**
+ * Optimize SPARQL query by reordering patterns
+ * @param {string} query - Original SPARQL query
+ * @param {Store} [store] - Optional store for statistics
+ * @returns {Object} Optimization result
+ *
+ * @throws {TypeError} If query is not a string
+ *
+ * @example
+ * const result = optimizeQuery(`
+ *   SELECT ?name WHERE {
+ *     ?person foaf:knows ?friend .
+ *     ?person foaf:name ?name .
+ *   }
+ * `);
+ *
+ * console.log('Optimized:', result.optimizedQuery);
+ * console.log('Speedup:', result.estimatedSpeedup);
+ */
+export function optimizeQuery(query, _store = null) {
+  if (typeof query !== 'string') {
+    throw new TypeError('optimizeQuery: query must be a string');
+  }
+
+  const analysis = analyzeSparqlQuery(query);
+  const changes = [];
+
+  // Strategy: Move most selective patterns first (filter early)
+  const optimizedPatterns = optimizePatternOrder(analysis.patterns, changes);
+
+  // Rebuild query with optimized pattern order
+  const optimizedQuery = rebuildQuery(query, analysis, optimizedPatterns);
+
+  // Estimate speedup
+  const estimatedSpeedup = calculateSpeedup(analysis.patterns, optimizedPatterns);
+
+  const result = {
+    originalQuery: query,
+    optimizedQuery,
+    changes,
+    estimatedSpeedup,
+  };
+
+  return OptimizationResultSchema.parse(result);
+}
+
+/**
+ * Optimize pattern execution order
+ * @param {Array} patterns - Original patterns
+ * @param {Array} changes - Changes accumulator
+ * @returns {Array} Optimized patterns
+ */
+function optimizePatternOrder(patterns, changes) {
+  if (patterns.length <= 1) {
+    return patterns;
+  }
+
+  // Sort by selectivity (descending) - execute most selective first
+  const sorted = [...patterns].sort((a, b) => b.selectivity - a.selectivity);
+
+  // Check if order changed
+  let orderChanged = false;
+  for (let i = 0; i < patterns.length; i++) {
+    if (patterns[i] !== sorted[i]) {
+      orderChanged = true;
+      break;
+    }
+  }
+
+  if (orderChanged) {
+    changes.push({
+      type: 'pattern_reorder',
+      description: 'Reordered patterns by selectivity (most selective first)',
+    });
+  }
+
+  return sorted;
+}
+
+/**
+ * Rebuild query with optimized patterns
+ * @param {string} originalQuery - Original query
+ * @param {Object} analysis - Query analysis
+ * @param {Array} optimizedPatterns - Optimized patterns
+ * @returns {string} Rebuilt query
+ */
+function rebuildQuery(originalQuery, analysis, optimizedPatterns) {
+  // Extract query parts
+  const whereMatch = originalQuery.match(/(.*WHERE\s*\{)([^}]+)(\}.*)/is);
+
+  if (!whereMatch) {
+    return originalQuery; // Can't optimize without WHERE clause
+  }
+
+  const [, before, , after] = whereMatch;
+
+  // Rebuild WHERE clause with optimized pattern order
+  const patternStrings = optimizedPatterns.map(p => `${p.subject} ${p.predicate} ${p.object}`);
+  const newWhereClause = patternStrings.join(' .\n    ');
+
+  return `${before}\n    ${newWhereClause}\n  ${after}`;
+}
+
+/**
+ * Calculate estimated speedup from optimization
+ * @param {Array} original - Original patterns
+ * @param {Array} optimized - Optimized patterns
+ * @returns {number} Estimated speedup multiplier
+ */
+function calculateSpeedup(original, optimized) {
+  // Speedup estimate based on selectivity improvement
+  // Early filtering reduces intermediate result size
+
+  let originalCost = 0;
+  let optimizedCost = 0;
+  let intermediateSize = 1000; // Base estimate
+
+  // Original order cost
+  for (const pattern of original) {
+    originalCost += intermediateSize;
+    intermediateSize *= 1 - pattern.selectivity; // Results after this pattern
+  }
+
+  // Reset for optimized
+  intermediateSize = 1000;
+
+  // Optimized order cost
+  for (const pattern of optimized) {
+    optimizedCost += intermediateSize;
+    intermediateSize *= 1 - pattern.selectivity;
+  }
+
+  const speedup = originalCost > 0 ? originalCost / Math.max(optimizedCost, 1) : 1.0;
+  return Math.round(speedup * 100) / 100;
+}
+
+/**
+ * Suggest indexes for store
+ * @param {Store} store - RDF store
+ * @param {string} query - SPARQL query
+ * @returns {Array<Object>} Index suggestions
+ *
+ * @throws {TypeError} If store or query is invalid
+ *
+ * @example
+ * const suggestions = suggestIndexes(store, query);
+ * suggestions.forEach(s => console.log(s.type, s.priority));
+ */
+export function suggestIndexes(store, query) {
+  if (!store || typeof store.getQuads !== 'function') {
+    throw new TypeError('suggestIndexes: store must be a valid Store instance');
+  }
+
+  if (typeof query !== 'string') {
+    throw new TypeError('suggestIndexes: query must be a string');
+  }
+
+  const analysis = analyzeSparqlQuery(query);
+  const suggestions = [];
+
+  // Track unique predicates to avoid duplicates
+  const uniquePredicates = new Set();
+
+  // Analyze patterns for index opportunities
+  for (const pattern of analysis.patterns) {
+    // Predicate-specific indexes
+    const cleanPredicate = pattern.predicate.trim();
+    if (!cleanPredicate.startsWith('?') && cleanPredicate.length > 0) {
+      if (!uniquePredicates.has(cleanPredicate)) {
+        uniquePredicates.add(cleanPredicate);
+        suggestions.push({
+          type: 'predicate_index',
+          predicate: cleanPredicate,
+          priority: 'high',
+          reason: 'Specific predicate queries benefit from indexes',
+        });
+      }
+    }
+
+    // Subject-predicate composite indexes for bound subjects
+    if (!pattern.subject.startsWith('?') && !cleanPredicate.startsWith('?')) {
+      suggestions.push({
+        type: 'subject_predicate_index',
+        subject: pattern.subject,
+        predicate: cleanPredicate,
+        priority: 'medium',
+        reason: 'Bound subject+predicate can use composite index',
+      });
+    }
+  }
+
+  // Join-based suggestions
+  for (const join of analysis.joins) {
+    if (join.patterns.length > 2) {
+      suggestions.push({
+        type: 'join_index',
+        variable: join.variable,
+        priority: 'medium',
+        reason: `Variable ${join.variable} joins ${join.patterns.length} patterns`,
+      });
+    }
+  }
+
+  return suggestions;
+}
+
+/**
+ * Explain optimization changes
+ * @param {string} original - Original query
+ * @param {string} optimized - Optimized query
+ * @returns {Object} Explanation
+ *
+ * @example
+ * const explanation = explainOptimization(original, optimized);
+ * console.log(explanation.summary);
+ */
+export function explainOptimization(original, optimized) {
+  if (typeof original !== 'string' || typeof optimized !== 'string') {
+    throw new TypeError('explainOptimization: queries must be strings');
+  }
+
+  const originalAnalysis = analyzeSparqlQuery(original);
+  const optimizedAnalysis = analyzeSparqlQuery(optimized);
+
+  const explanation = {
+    summary: 'Query optimization analysis',
+    originalComplexity: originalAnalysis.patterns.length,
+    optimizedComplexity: optimizedAnalysis.patterns.length,
+    changes: [],
+  };
+
+  // Compare pattern order
+  let patternOrderChanged = false;
+  for (
+    let i = 0;
+    i < Math.min(originalAnalysis.patterns.length, optimizedAnalysis.patterns.length);
+    i++
+  ) {
+    if (
+      JSON.stringify(originalAnalysis.patterns[i]) !== JSON.stringify(optimizedAnalysis.patterns[i])
+    ) {
+      patternOrderChanged = true;
+      break;
+    }
+  }
+
+  if (patternOrderChanged) {
+    explanation.changes.push({
+      type: 'pattern_order',
+      description: 'Patterns reordered for better selectivity',
+      impact: 'Reduces intermediate result set size',
+    });
+  }
+
+  return explanation;
+}