magector 1.0.0

package/src/validation/test-queries.js

@@ -0,0 +1,326 @@
+/**
+ * Test queries with expected results for accuracy validation
+ */
+
+export const TEST_QUERIES = [
+  // Controller queries
+  {
+    id: 'ctrl-1',
+    query: 'controller execute action',
+    type: 'semantic',
+    expectedTypes: ['Controller'],
+    expectedPatterns: ['controller'],
+    minResults: 1,
+    category: 'controller'
+  },
+  {
+    id: 'ctrl-2',
+    query: 'Index controller',
+    type: 'exact',
+    expectedClasses: ['Index'],
+    expectedTypes: ['Controller'],
+    category: 'controller'
+  },
+
+  // Model queries
+  {
+    id: 'model-1',
+    query: 'model beforeSave afterLoad',
+    type: 'semantic',
+    expectedTypes: ['Model'],
+    expectedPatterns: ['model'],
+    category: 'model'
+  },
+  {
+    id: 'model-2',
+    query: 'AbstractModel extends',
+    type: 'semantic',
+    expectedInContent: ['extends AbstractModel'],
+    category: 'model'
+  },
+
+  // Repository queries
+  {
+    id: 'repo-1',
+    query: 'repository getById save delete',
+    type: 'semantic',
+    expectedTypes: ['Repository'],
+    expectedPatterns: ['repository'],
+    category: 'repository'
+  },
+  {
+    id: 'repo-2',
+    query: 'ItemRepository',
+    type: 'exact',
+    expectedClasses: ['ItemRepository'],
+    category: 'repository'
+  },
+  {
+    id: 'repo-3',
+    query: 'getList SearchCriteria',
+    type: 'semantic',
+    expectedInContent: ['getList', 'SearchCriteria'],
+    category: 'repository'
+  },
+
+  // Plugin queries
+  {
+    id: 'plugin-1',
+    query: 'plugin interceptor before after around',
+    type: 'semantic',
+    expectedTypes: ['Plugin'],
+    expectedPatterns: ['plugin'],
+    category: 'plugin'
+  },
+  {
+    id: 'plugin-2',
+    query: 'beforeGetPrice plugin',
+    type: 'semantic',
+    expectedInContent: ['beforeGetPrice', 'before'],
+    category: 'plugin'
+  },
+  {
+    id: 'plugin-3',
+    query: 'around method interceptor',
+    type: 'semantic',
+    expectedInContent: ['around', 'proceed'],
+    category: 'plugin'
+  },
+
+  // Observer queries
+  {
+    id: 'obs-1',
+    query: 'observer execute event',
+    type: 'semantic',
+    expectedTypes: ['Observer'],
+    expectedPatterns: ['observer'],
+    category: 'observer'
+  },
+  {
+    id: 'obs-2',
+    query: 'catalog_product_save_after observer',
+    type: 'semantic',
+    expectedInContent: ['catalog_product_save_after'],
+    category: 'observer'
+  },
+  {
+    id: 'obs-3',
+    query: 'ObserverInterface implement',
+    type: 'semantic',
+    expectedInContent: ['ObserverInterface'],
+    category: 'observer'
+  },
+
+  // Block queries
+  {
+    id: 'block-1',
+    query: 'block template phtml',
+    type: 'semantic',
+    expectedTypes: ['Block'],
+    expectedPatterns: ['block'],
+    category: 'block'
+  },
+  {
+    id: 'block-2',
+    query: '_toHtml _prepareLayout',
+    type: 'semantic',
+    expectedInContent: ['_toHtml', '_prepareLayout'],
+    category: 'block'
+  },
+
+  // DI.xml queries
+  {
+    id: 'di-1',
+    query: 'preference interface implementation',
+    type: 'semantic',
+    expectedFileTypes: ['xml'],
+    expectedInContent: ['preference'],
+    category: 'di'
+  },
+  {
+    id: 'di-2',
+    query: 'plugin configuration di.xml',
+    type: 'semantic',
+    expectedFileTypes: ['xml'],
+    expectedInContent: ['plugin'],
+    category: 'di'
+  },
+  {
+    id: 'di-3',
+    query: 'virtualType argument',
+    type: 'semantic',
+    expectedInContent: ['virtualType'],
+    category: 'di'
+  },
+
+  // Events.xml queries
+  {
+    id: 'event-1',
+    query: 'event observer configuration',
+    type: 'semantic',
+    expectedFileTypes: ['xml'],
+    expectedInContent: ['event', 'observer'],
+    category: 'events'
+  },
+  {
+    id: 'event-2',
+    query: 'catalog_product_save_after event config',
+    type: 'exact',
+    expectedInContent: ['catalog_product_save_after'],
+    category: 'events'
+  },
+
+  // Web API queries
+  {
+    id: 'api-1',
+    query: 'REST API endpoint route',
+    type: 'semantic',
+    expectedFileTypes: ['xml'],
+    expectedInContent: ['route', 'service'],
+    category: 'webapi'
+  },
+  {
+    id: 'api-2',
+    query: 'GET /V1/items',
+    type: 'semantic',
+    expectedInContent: ['GET', '/V1/'],
+    category: 'webapi'
+  },
+
+  // GraphQL queries
+  {
+    id: 'gql-1',
+    query: 'GraphQL resolver',
+    type: 'semantic',
+    expectedTypes: ['GraphQlResolver'],
+    expectedPatterns: ['graphql_resolver'],
+    category: 'graphql'
+  },
+  {
+    id: 'gql-2',
+    query: 'type Query mutation',
+    type: 'semantic',
+    expectedFileTypes: ['graphql'],
+    expectedInContent: ['type', 'Query'],
+    category: 'graphql'
+  },
+  {
+    id: 'gql-3',
+    query: 'ResolverInterface resolve Field',
+    type: 'semantic',
+    expectedInContent: ['ResolverInterface', 'resolve'],
+    category: 'graphql'
+  },
+
+  // Cron queries
+  {
+    id: 'cron-1',
+    query: 'cron job schedule execute',
+    type: 'semantic',
+    expectedTypes: ['Cron'],
+    category: 'cron'
+  },
+  {
+    id: 'cron-2',
+    query: 'crontab.xml job instance',
+    type: 'semantic',
+    expectedFileTypes: ['xml'],
+    expectedInContent: ['job', 'instance'],
+    category: 'cron'
+  },
+
+  // Cross-cutting queries
+  {
+    id: 'cross-1',
+    query: 'dependency injection constructor',
+    type: 'semantic',
+    expectedInContent: ['__construct'],
+    minResults: 3,
+    category: 'di_pattern'
+  },
+  {
+    id: 'cross-2',
+    query: 'LoggerInterface logging',
+    type: 'semantic',
+    expectedInContent: ['LoggerInterface', 'logger'],
+    category: 'logging'
+  },
+  {
+    id: 'cross-3',
+    query: 'exception handling try catch',
+    type: 'semantic',
+    expectedInContent: ['Exception', 'throw'],
+    category: 'error_handling'
+  },
+
+  // Module-specific queries
+  {
+    id: 'mod-1',
+    query: 'Acme_Catalog module',
+    type: 'module',
+    expectedModule: 'Acme_Catalog',
+    minResults: 3,
+    category: 'module'
+  },
+
+  // Method-specific queries
+  {
+    id: 'method-1',
+    query: 'function getById',
+    type: 'method',
+    expectedMethods: ['getById'],
+    category: 'method'
+  },
+  {
+    id: 'method-2',
+    query: 'save method repository',
+    type: 'method',
+    expectedMethods: ['save'],
+    category: 'method'
+  }
+];
+
+export const QUERY_CATEGORIES = {
+  controller: { weight: 1.0, description: 'Controller action detection' },
+  model: { weight: 1.0, description: 'Model and lifecycle hooks' },
+  repository: { weight: 1.2, description: 'Repository pattern detection' },
+  plugin: { weight: 1.3, description: 'Plugin/interceptor detection' },
+  observer: { weight: 1.2, description: 'Observer pattern detection' },
+  block: { weight: 1.0, description: 'Block class detection' },
+  di: { weight: 1.2, description: 'DI configuration detection' },
+  events: { weight: 1.1, description: 'Event configuration detection' },
+  webapi: { weight: 1.2, description: 'Web API route detection' },
+  graphql: { weight: 1.3, description: 'GraphQL schema/resolver detection' },
+  cron: { weight: 1.0, description: 'Cron job detection' },
+  di_pattern: { weight: 0.8, description: 'DI pattern recognition' },
+  logging: { weight: 0.7, description: 'Logging pattern recognition' },
+  error_handling: { weight: 0.7, description: 'Error handling patterns' },
+  module: { weight: 1.0, description: 'Module filtering' },
+  method: { weight: 1.0, description: 'Method search' }
+};
+
+/**
+ * Generate edge case queries for stress testing
+ */
+export function generateEdgeCaseQueries() {
+  return [
+    // Very short queries
+    { id: 'edge-1', query: 'save', type: 'short', category: 'edge_short', minResults: 1, expectedMethods: ['save'] },
+    { id: 'edge-2', query: 'get', type: 'short', category: 'edge_short', minResults: 1 },
+
+    // Very long queries
+    { id: 'edge-3', query: 'public function execute action controller', type: 'long', category: 'edge_long', minResults: 1, expectedPatterns: ['controller'] },
+
+    // Technical jargon
+    { id: 'edge-4', query: 'CRUD operations repository interface', type: 'jargon', category: 'edge_jargon', minResults: 1, expectedTypes: ['Repository'] },
+
+    // Magento-specific terms - lower expectations
+    { id: 'edge-5', query: 'service contract API', type: 'magento_specific', category: 'edge_magento', minResults: 1 },
+
+    // Negative queries (should return few/no results)
+    { id: 'edge-6', query: 'wordpress drupal laravel', type: 'negative', category: 'edge_negative', maxResults: 2 },
+
+    // Mixed case
+    { id: 'edge-7', query: 'REPOSITORY getbyid SAVE', type: 'case', category: 'edge_case', minResults: 1, expectedTypes: ['Repository'] }
+  ];
+}

package/src/validation/validator.js

@@ -0,0 +1,302 @@
+#!/usr/bin/env node
+/**
+ * Main validation runner
+ */
+
+import { MagentoIndexer } from '../indexer.js';
+import { generateCompleteMockModule, MOCK_MODULES } from './test-data-generator.js';
+import { TEST_QUERIES, QUERY_CATEGORIES, generateEdgeCaseQueries } from './test-queries.js';
+import {
+  calculatePrecision,
+  calculateRecall,
+  calculateF1,
+  calculateMRR,
+  calculateNDCG,
+  aggregateMetrics,
+  gradeAccuracy,
+  generateReport,
+  isResultRelevant
+} from './accuracy-calculator.js';
+import { writeFile, mkdir, rm } from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+
+const VALIDATION_DIR = '/private/tmp/magector-validation';
+const RESULTS_DIR = './validation-results';
+
+export class MagectorValidator {
+  constructor(options = {}) {
+    this.indexer = null;
+    this.testData = [];
+    this.results = [];
+    this.verbose = options.verbose || false;
+  }
+
+  async setup() {
+    console.log('Setting up validation environment...\n');
+
+    // Clean up previous test data
+    if (existsSync(VALIDATION_DIR)) {
+      await rm(VALIDATION_DIR, { recursive: true });
+    }
+    await mkdir(VALIDATION_DIR, { recursive: true });
+    await mkdir(`${VALIDATION_DIR}/app/code`, { recursive: true });
+
+    // Generate mock modules
+    console.log('Generating mock Magento modules...');
+    for (const moduleName of MOCK_MODULES) {
+      const files = generateCompleteMockModule(moduleName);
+      this.testData.push(...files);
+
+      for (const file of files) {
+        const filePath = path.join(VALIDATION_DIR, file.path);
+        const dir = path.dirname(filePath);
+        await mkdir(dir, { recursive: true });
+        await writeFile(filePath, file.content);
+      }
+      console.log(`  ✓ Generated ${moduleName} (${files.length} files)`);
+    }
+
+    console.log(`\nTotal test files: ${this.testData.length}`);
+
+    // Initialize indexer
+    console.log('\nInitializing indexer with GNN...');
+    this.indexer = new MagentoIndexer({
+      dbPath: `${VALIDATION_DIR}/magector-test.db`,
+      graphPath: `${VALIDATION_DIR}/magector-test-graph.json`,
+      magentoRoot: VALIDATION_DIR,
+      enableGNN: true
+    });
+    await this.indexer.init();
+
+    // Index the test data
+    console.log('Indexing test data...');
+    const stats = await this.indexer.indexDirectory(VALIDATION_DIR);
+    console.log(`  Indexed: ${stats.indexed} files`);
+    console.log(`  Graph nodes: ${stats.graphNodes}`);
+    console.log(`  Graph edges: ${stats.graphEdges}\n`);
+
+    return stats;
+  }
+
+  async runQuery(testQuery) {
+    const startTime = Date.now();
+
+    try {
+      const results = await this.indexer.searchWithGraph(testQuery.query, { limit: 10 });
+      const duration = Date.now() - startTime;
+
+      // Calculate expected count based on test data
+      const expectedCount = this.testData.filter(td =>
+        isResultRelevant({ ...td.metadata, content: td.content }, testQuery)
+      ).length;
+
+      const precision = calculatePrecision(results, testQuery);
+      const recall = calculateRecall(results, testQuery, Math.max(expectedCount, testQuery.minResults || 1));
+      const f1 = calculateF1(precision, recall);
+      const mrr = calculateMRR(results, testQuery);
+      const ndcg = calculateNDCG(results, testQuery);
+
+      // Determine pass/fail
+      let passed = true;
+      let failReason = '';
+
+      if (testQuery.minResults && results.length < testQuery.minResults) {
+        passed = false;
+        failReason = `Expected min ${testQuery.minResults} results, got ${results.length}`;
+      } else if (testQuery.maxResults !== undefined && results.length > testQuery.maxResults) {
+        passed = false;
+        failReason = `Expected max ${testQuery.maxResults} results, got ${results.length}`;
+      } else if (precision < 0.3) {
+        passed = false;
+        failReason = `Precision too low: ${(precision * 100).toFixed(1)}%`;
+      } else if (results.length > 0 && mrr === 0) {
+        passed = false;
+        failReason = 'No relevant results in top 10';
+      }
+
+      return {
+        queryId: testQuery.id,
+        query: testQuery.query,
+        category: testQuery.category || 'unknown',
+        resultCount: results.length,
+        expectedCount,
+        precision,
+        recall,
+        f1,
+        mrr,
+        ndcg,
+        duration,
+        passed,
+        failReason,
+        topResults: results.slice(0, 3).map(r => ({
+          path: r.path,
+          type: r.magentoType || r.type,
+          score: r.score,
+          relevant: isResultRelevant(r, testQuery)
+        }))
+      };
+    } catch (error) {
+      return {
+        queryId: testQuery.id,
+        query: testQuery.query,
+        category: testQuery.category || 'unknown',
+        passed: false,
+        failReason: `Error: ${error.message}`,
+        precision: 0,
+        recall: 0,
+        f1: 0,
+        mrr: 0,
+        ndcg: 0,
+        duration: Date.now() - startTime
+      };
+    }
+  }
+
+  async runAllQueries() {
+    console.log('Running validation queries...\n');
+    const allQueries = [...TEST_QUERIES, ...generateEdgeCaseQueries()];
+
+    let completed = 0;
+    for (const query of allQueries) {
+      const result = await this.runQuery(query);
+      this.results.push(result);
+      completed++;
+
+      if (this.verbose) {
+        const status = result.passed ? '✓' : '✗';
+        console.log(`  ${status} [${query.id}] ${query.query.substring(0, 40).padEnd(40)} F1: ${(result.f1 * 100).toFixed(1).padStart(5)}%`);
+      } else if (completed % 10 === 0) {
+        process.stdout.write(`  Progress: ${completed}/${allQueries.length}\r`);
+      }
+    }
+
+    console.log(`\nCompleted ${completed} queries.`);
+    return this.results;
+  }
+
+  async runCategoryBenchmark(category) {
+    const queries = TEST_QUERIES.filter(q => q.category === category);
+    const results = [];
+
+    for (const query of queries) {
+      results.push(await this.runQuery(query));
+    }
+
+    return aggregateMetrics(results);
+  }
+
+  async runLatencyBenchmark(iterations = 100) {
+    console.log(`\nRunning latency benchmark (${iterations} iterations)...`);
+
+    const latencies = [];
+    const sampleQueries = TEST_QUERIES.slice(0, 10);
+
+    for (let i = 0; i < iterations; i++) {
+      const query = sampleQueries[i % sampleQueries.length];
+      const start = Date.now();
+      await this.indexer.search(query.query, { limit: 10 });
+      latencies.push(Date.now() - start);
+    }
+
+    latencies.sort((a, b) => a - b);
+
+    return {
+      min: latencies[0],
+      max: latencies[latencies.length - 1],
+      avg: latencies.reduce((a, b) => a + b, 0) / latencies.length,
+      p50: latencies[Math.floor(latencies.length * 0.5)],
+      p90: latencies[Math.floor(latencies.length * 0.9)],
+      p99: latencies[Math.floor(latencies.length * 0.99)]
+    };
+  }
+
+  async generateFullReport() {
+    const metrics = aggregateMetrics(this.results);
+    const grade = gradeAccuracy(metrics);
+    const report = generateReport(metrics, grade);
+
+    // Ensure results directory exists
+    await mkdir(RESULTS_DIR, { recursive: true });
+
+    // Save detailed results
+    const detailedResults = {
+      timestamp: new Date().toISOString(),
+      metrics,
+      grade,
+      results: this.results
+    };
+
+    await writeFile(
+      `${RESULTS_DIR}/validation-results.json`,
+      JSON.stringify(detailedResults, null, 2)
+    );
+
+    await writeFile(`${RESULTS_DIR}/validation-report.txt`, report);
+
+    return { metrics, grade, report };
+  }
+
+  async cleanup() {
+    if (existsSync(VALIDATION_DIR)) {
+      await rm(VALIDATION_DIR, { recursive: true });
+    }
+  }
+}
+
+/**
+ * Run full validation suite
+ */
+export async function runFullValidation(options = {}) {
+  const validator = new MagectorValidator(options);
+
+  try {
+    console.log('╔════════════════════════════════════════════════════════════════╗');
+    console.log('║           MAGECTOR ACCURACY VALIDATION SUITE                   ║');
+    console.log('╚════════════════════════════════════════════════════════════════╝\n');
+
+    // Setup
+    await validator.setup();
+
+    // Run queries
+    await validator.runAllQueries();
+
+    // Latency benchmark
+    const latency = await validator.runLatencyBenchmark();
+    console.log('\nLatency Results:');
+    console.log(`  Min: ${latency.min}ms, Avg: ${latency.avg.toFixed(1)}ms, P99: ${latency.p99}ms`);
+
+    // Generate report
+    const { metrics, grade, report } = await validator.generateFullReport();
+
+    // Print report
+    console.log(report);
+
+    // Print summary
+    console.log('Results saved to:');
+    console.log('  - validation-results/validation-report.txt');
+    console.log('  - validation-results/validation-results.json\n');
+
+    return { metrics, grade, latency };
+
+  } finally {
+    if (!options.keepTestData) {
+      await validator.cleanup();
+    }
+  }
+}
+
+// CLI execution
+if (process.argv[1].includes('validator.js')) {
+  const verbose = process.argv.includes('--verbose') || process.argv.includes('-v');
+  const keepData = process.argv.includes('--keep');
+
+  runFullValidation({ verbose, keepTestData: keepData })
+    .then(({ grade }) => {
+      process.exit(grade.score >= 70 ? 0 : 1);
+    })
+    .catch(err => {
+      console.error('Validation failed:', err);
+      process.exit(1);
+    });
+}