@crownpeak/dqm-react-component-dev-mcp 1.2.0

package/data/probe/tests/symbols_tests.rs

@@ -0,0 +1,166 @@
+use probe_code::search::{perform_probe, SearchOptions};
+use std::fs;
+use tempfile::tempdir;
+
+#[test]
+fn test_symbols_flag_with_rust_code() {
+    // Create a temporary directory and file
+    let temp_dir = tempdir().expect("Failed to create temp dir");
+    let test_file = temp_dir.path().join("test.rs");
+
+    // Write Rust code with various symbols
+    let rust_code = r#"
+pub struct User {
+    pub name: String,
+    pub age: u32,
+}
+
+impl User {
+    pub fn new(name: String, age: u32) -> Self {
+        Self { name, age }
+    }
+
+    pub fn greet(&self) -> String {
+        format!("Hello, I'm {}", self.name)
+    }
+}
+
+pub fn main() {
+    let user = User::new("Alice".to_string(), 30);
+    println!("{}", user.greet());
+}
+
+pub const MAX_USERS: usize = 1000;
+"#;
+
+    fs::write(&test_file, rust_code).expect("Failed to write test file");
+
+    // Test search with symbols flag enabled
+    let options = SearchOptions {
+        path: temp_dir.path(),
+        queries: &["pub fn".to_string()],
+        files_only: false,
+        custom_ignores: &[],
+        exclude_filenames: false,
+        reranker: "bm25",
+        frequency_search: true,
+        exact: false,
+        language: None,
+        max_results: Some(10),
+        max_bytes: None,
+        max_tokens: None,
+        allow_tests: false,
+        no_merge: false,
+        merge_threshold: None,
+        dry_run: false,
+        session: None,
+        timeout: 30,
+        question: None,
+        no_gitignore: true,
+    };
+
+    let results = perform_probe(&options).expect("Search should succeed");
+
+    // Verify we got results
+    assert!(!results.results.is_empty(), "Should find symbol matches");
+
+    // Verify that symbol signatures are populated
+    let has_symbol_signatures = results.results.iter().any(|r| r.symbol_signature.is_some());
+    assert!(
+        has_symbol_signatures,
+        "At least one result should have a symbol signature"
+    );
+
+    // Test with symbols flag disabled
+    let options_no_symbols = SearchOptions { ..options };
+
+    let results_no_symbols = perform_probe(&options_no_symbols).expect("Search should succeed");
+
+    // Verify that symbol signatures are not populated when flag is disabled
+    let has_symbol_signatures = results_no_symbols
+        .results
+        .iter()
+        .any(|r| r.symbol_signature.is_some());
+    assert!(
+        !has_symbol_signatures,
+        "No results should have symbol signatures when flag is disabled"
+    );
+}
+
+#[test]
+fn test_symbols_flag_with_python_code() {
+    // Create a temporary directory and file
+    let temp_dir = tempdir().expect("Failed to create temp dir");
+    let test_file = temp_dir.path().join("test.py");
+
+    // Write Python code with various symbols
+    let python_code = r#"
+class User:
+    def __init__(self, name: str, age: int):
+        self.name = name
+        self.age = age
+
+    def greet(self) -> str:
+        return f"Hello, I'm {self.name}"
+
+def create_user(name: str, age: int) -> User:
+    return User(name, age)
+
+async def async_function(data: list) -> dict:
+    return {"length": len(data)}
+
+MAX_USERS = 1000
+"#;
+
+    fs::write(&test_file, python_code).expect("Failed to write test file");
+
+    // Test search with symbols flag enabled
+    let options = SearchOptions {
+        path: temp_dir.path(),
+        queries: &["def ".to_string()],
+        files_only: false,
+        custom_ignores: &[],
+        exclude_filenames: false,
+        reranker: "bm25",
+        frequency_search: true,
+        exact: false,
+        language: None,
+        max_results: Some(10),
+        max_bytes: None,
+        max_tokens: None,
+        allow_tests: false,
+        no_merge: false,
+        merge_threshold: None,
+        dry_run: false,
+        session: None,
+        timeout: 30,
+        question: None,
+        no_gitignore: true,
+    };
+
+    let results = perform_probe(&options).expect("Search should succeed");
+
+    // Verify we got results
+    assert!(!results.results.is_empty(), "Should find symbol matches");
+
+    // Verify that symbol signatures are populated and contain Python syntax
+    let python_symbols: Vec<_> = results
+        .results
+        .iter()
+        .filter_map(|r| r.symbol_signature.as_ref())
+        .collect();
+
+    assert!(
+        !python_symbols.is_empty(),
+        "Should have Python symbol signatures"
+    );
+
+    // Check that we have recognizable Python function signatures
+    let has_python_function = python_symbols
+        .iter()
+        .any(|sig| sig.contains("def ") && sig.contains("(") && sig.contains(")"));
+    assert!(
+        has_python_function,
+        "Should have Python function signatures"
+    );
+}

package/data/probe/tests/test_file.rs

@@ -0,0 +1,45 @@
+// This is a test file for probe
+
+#[allow(dead_code)]
+fn function1() {
+    println!("This is function 1");
+    // Some code here
+    let x = 10;
+    let y = 20;
+    println!("Sum: {}", x + y);
+}
+
+// This comment is between function1 and function2
+// Only a few lines of separation
+
+#[allow(dead_code)]
+fn function2() {
+    println!("This is function 2");
+    // Some code here
+    let a = 30;
+    let b = 40;
+    println!("Product: {}", a * b);
+}
+
+// This is a larger gap between functions
+
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+// More comments
+
+#[allow(dead_code)]
+fn function3() {
+    println!("This is function 3");
+    // Some code here
+    let c = 50;
+    let d = 60;
+    println!("Difference: {}", d - c);
+}

package/data/probe/tests/test_tokenize.rs

@@ -0,0 +1,28 @@
+fn main() {
+    // Import the tokenize function from our probe crate
+    use probe_code::ranking::tokenize;
+
+    // Test strings
+    let test_strings = ["The quick brown fox jumps over the lazy dog",
+        "function calculateTotal(items) { return items.reduce((sum, item) => sum + item.price, 0); }",
+        "class UserController extends BaseController implements UserInterface",
+        "Searching for files containing important information",
+        "Fruitlessly searching for the missing variable in the codebase"];
+
+    println!("Testing tokenization with stop word removal and stemming:\n");
+
+    for (i, test_str) in test_strings.iter().enumerate() {
+        println!("Original text {}:\n{}", i + 1, test_str);
+
+        // Tokenize with stop word removal and stemming
+        let tokens = tokenize(test_str);
+
+        println!("Tokens after stop word removal and stemming:");
+        println!("{tokens:?}");
+        println!("Number of tokens: {}\n", tokens.len());
+    }
+
+    // Specific test for stemming
+    println!("Specific stemming test:");
+    println!("'fruitlessly' stems to: {}", tokenize("fruitlessly")[0]);
+}

package/data/probe/tests/timeout_tests.rs

@@ -0,0 +1,82 @@
+use std::path::Path;
+use std::process::Command;
+use std::time::Instant;
+use tempfile::TempDir;
+
+/// Create a large test file that will take time to process
+fn create_large_test_file(temp_dir: &Path) {
+    let file_path = temp_dir.join("large_file.rs");
+
+    // Create a very large file with many lines to ensure the search takes longer than the timeout
+    let mut content = String::new();
+    for i in 0..500000 {
+        // Increased to 500,000 lines
+        content.push_str(&format!("// Line {i} with searchable content\n"));
+        content.push_str(&format!("fn function_{i}() {{\n"));
+        content.push_str(&format!("    let search_term = {i};\n"));
+        content.push_str("    println!(\"Found: {}\", search_term);\n");
+        content.push_str("}\n\n");
+    }
+
+    // Write the file
+    std::fs::write(file_path, content).expect("Failed to write large test file");
+}
+
+/// Test that the search operation times out after the specified timeout
+/// This test runs the search command in a separate process since our timeout
+/// implementation calls std::process::exit(1)
+#[test]
+fn test_search_timeout() {
+    // Create a temporary directory
+    let temp_dir = TempDir::new().expect("Failed to create temp dir");
+
+    // Create a large test file
+    create_large_test_file(temp_dir.path());
+
+    // Measure the time it takes to run the search
+    let start_time = Instant::now();
+
+    // Run the search command with a very short timeout using cargo run
+    let output = Command::new("cargo")
+        .arg("run")
+        .arg("--")
+        .arg("search")
+        .arg("search_term") // This term appears in every function
+        .arg("/")
+        .arg("--timeout")
+        .arg("1") // 1 second timeout should be short enough to trigger
+        .output()
+        .expect("Failed to execute command");
+
+    // Check how long it took
+    let elapsed = start_time.elapsed();
+
+    // Print the output for debugging
+    println!(
+        "Command output: {}",
+        String::from_utf8_lossy(&output.stdout)
+    );
+    println!("Command error: {}", String::from_utf8_lossy(&output.stderr));
+
+    // The command should have failed (non-zero exit code)
+    assert_ne!(
+        output.status.code().unwrap_or(-1),
+        0,
+        "Command should have failed with a non-zero exit code"
+    );
+
+    // The error output should mention timeout
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    assert!(
+        stderr.contains("timeout") || stderr.contains("timed out"),
+        "Error output should mention timeout: {stderr}"
+    );
+
+    // The elapsed time should be close to the timeout value
+    assert!(
+        elapsed.as_secs() >= 1 && elapsed.as_secs() <= 5,
+        "Search should have timed out after about 1 second, but took {elapsed:?}"
+    );
+
+    println!("✓ Search timed out correctly after {elapsed:?}");
+}

package/data/probe/tests/tokenization_tests.rs

@@ -0,0 +1,195 @@
+use probe_code::search::elastic_query::parse_query_test as parse_query;
+
+use probe_code::search::query::{create_query_plan, create_structured_patterns};
+use probe_code::search::tokenization::{load_vocabulary, split_camel_case, split_compound_word};
+
+#[test]
+fn test_camel_case_splitting() {
+    // Test camelCase splitting with "RPCStorageHandler"
+    let input = "RPCStorageHandler";
+    let camel_parts = split_camel_case(input);
+
+    println!("Direct camelCase split of '{input}': {camel_parts:?}");
+
+    // We expect "RPCStorageHandler" to be split into ["rpc", "storage", "handler"]
+    assert!(camel_parts.contains(&"rpc".to_string()));
+    assert!(camel_parts.contains(&"storage".to_string()));
+    assert!(camel_parts.contains(&"handler".to_string()));
+}
+
+#[test]
+fn test_compound_word_splitting() {
+    // Test compound word splitting with "whitelist"
+    let vocab = load_vocabulary();
+    let mut enhanced_vocab = vocab.clone();
+
+    // Add specific programming terms to the vocabulary
+    for term in [
+        "rpc", "storage", "handler", "client", "server", "api", "service", "http", "handler",
+    ] {
+        enhanced_vocab.insert(term.to_string());
+    }
+
+    let input = "httpHandler";
+    let compound_parts = split_compound_word(input, &enhanced_vocab);
+
+    println!("Compound split of '{input}': {compound_parts:?}");
+
+    assert!(compound_parts.contains(&"http".to_string()));
+    assert!(compound_parts.contains(&"Handler".to_string()));
+}
+
+#[test]
+fn test_query_preprocessing() {
+    // Test query preprocessing with "RPCStorageHandler"
+    let query = "RPCStorageHandler";
+    let plan = create_query_plan(query, false).expect("Failed to create query plan");
+
+    println!("Query plan for '{query}': {plan:?}");
+
+    // Check if the term_indices include the expected parts
+    let term_strings: Vec<String> = plan.term_indices.keys().cloned().collect();
+
+    // We expect to see "rpc", "storage", "handler" in the terms
+    assert!(
+        term_strings.contains(&"rpc".to_string())
+            || term_strings.contains(&"storage".to_string())
+            || term_strings.contains(&"handler".to_string()),
+        "Expected at least one of 'rpc', 'storage', or 'handler' in {term_strings:?}"
+    );
+}
+
+#[test]
+fn test_pattern_generation() {
+    // Test pattern generation with "RPCStorageHandler"
+    let query = "RPCStorageHandler";
+    let plan = create_query_plan(query, false).expect("Failed to create query plan");
+    let patterns = create_structured_patterns(&plan);
+
+    println!("Generated patterns:");
+    for (i, (pattern, indices)) in patterns.iter().enumerate() {
+        println!("Pattern {i}: {pattern} - Indices: {indices:?}");
+    }
+
+    // Check that we have patterns for the expected terms
+    let term_indices = &plan.term_indices;
+
+    // Verify that each term in the plan has at least one pattern
+    for (term, &idx) in term_indices {
+        if !plan.excluded_terms.contains(term) {
+            let has_pattern = patterns.iter().any(|(_, indices)| indices.contains(&idx));
+            assert!(
+                has_pattern,
+                "No pattern found for term '{term}' at index {idx}"
+            );
+        }
+    }
+
+    // Check that all patterns are valid regexes
+    for (pattern, _) in &patterns {
+        assert!(
+            regex::Regex::new(pattern).is_ok(),
+            "Invalid regex pattern: {pattern}"
+        );
+    }
+}
+
+#[test]
+fn test_multiple_word_query() {
+    // Test multiple words with "ip whitelist"
+    let query = "ip whitelist";
+    let plan = create_query_plan(query, false).expect("Failed to create query plan");
+
+    println!("Query plan for '{query}': {plan:?}");
+
+    // Check if the terms include the expected parts
+    let term_strings: Vec<String> = plan.term_indices.keys().cloned().collect();
+
+    // We expect to see "ip", "white", "list" in the terms
+    assert!(term_strings.contains(&"ip".to_string()));
+
+    // Either "white" and "list" separately, or "whitelist" as a whole
+    let has_white_and_list =
+        term_strings.contains(&"white".to_string()) && term_strings.contains(&"list".to_string());
+    let has_whitelist = term_strings.contains(&"whitelist".to_string());
+
+    assert!(
+        has_white_and_list || has_whitelist,
+        "Expected either both 'white' and 'list', or 'whitelist' in {term_strings:?}"
+    );
+}
+
+#[test]
+fn test_underscore_handling() {
+    use probe_code::search::elastic_query;
+
+    // Test tokenization with underscores
+    let query = "keyword_underscore";
+    let plan = create_query_plan(query, false).expect("Failed to create query plan");
+
+    println!("Query plan for '{query}': {plan:?}");
+
+    // Check if the term is preserved with the underscore in the term_indices
+    let term_strings: Vec<String> = plan.term_indices.keys().cloned().collect();
+
+    // We expect to see "keyword_underscore" or its tokenized parts
+    let has_tokenized_parts = term_strings.contains(&"key".to_string())
+        || term_strings.contains(&"word".to_string())
+        || term_strings.contains(&"score".to_string());
+
+    assert!(
+        has_tokenized_parts,
+        "Expected tokenized parts of 'keyword_underscore' in {term_strings:?}"
+    );
+
+    // Check if the term is properly tokenized in the elastic query parser
+    // This is already done in the create_query_plan function, but we'll verify the AST
+    if let elastic_query::Expr::Term { keywords, .. } = &plan.ast {
+        // The term should be tokenized into ["key", "word", "under", "score"]
+        // Note: "under" might be filtered out as a stop word
+        assert!(
+            keywords.contains(&"key".to_string()),
+            "Expected 'key' in keywords: {keywords:?}"
+        );
+        assert!(
+            keywords.contains(&"word".to_string()),
+            "Expected 'word' in keywords: {keywords:?}"
+        );
+        assert!(
+            keywords.contains(&"score".to_string()),
+            "Expected 'score' in keywords: {keywords:?}"
+        );
+    } else {
+        panic!("Expected Term expression");
+    }
+}
+
+#[test]
+fn test_underscore_in_elastic_query() {
+    use probe_code::search::elastic_query;
+
+    // Test that the elastic query parser preserves underscores
+    let query = "keyword_underscore";
+    // Use parse_query with standard Elasticsearch behavior (AND for implicit combinations)
+    let ast = parse_query(query).unwrap();
+
+    // Check that the AST contains the tokenized terms from "keyword_underscore"
+    if let elastic_query::Expr::Term { keywords, .. } = ast {
+        // The term should be tokenized into ["key", "word", "under", "score"]
+        // Note: "under" might be filtered out as a stop word
+        assert!(
+            keywords.contains(&"key".to_string()),
+            "Expected 'key' in keywords: {keywords:?}"
+        );
+        assert!(
+            keywords.contains(&"word".to_string()),
+            "Expected 'word' in keywords: {keywords:?}"
+        );
+        assert!(
+            keywords.contains(&"score".to_string()),
+            "Expected 'score' in keywords: {keywords:?}"
+        );
+    } else {
+        panic!("Expected Term expression");
+    }
+}

package/data/probe/tests/tokenized_block_filtering_tests.rs

@@ -0,0 +1,174 @@
+use probe_code::search::file_processing::filter_tokenized_block;
+use probe_code::search::query::create_query_plan;
+use std::collections::{HashMap, HashSet};
+
+/// Test direct usage of filter_tokenized_block with various complex queries
+#[test]
+fn test_filter_tokenized_block_basic() {
+    // Create a simple tokenized content
+    let tokenized_content = vec![
+        "ip".to_string(),
+        "whitelist".to_string(),
+        "config".to_string(),
+    ];
+
+    // Test with a simple AND query
+    let query = "ip AND whitelist";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(
+        result,
+        "Block with 'ip' and 'whitelist' should match the AND query"
+    );
+
+    // Test with a simple OR query
+    let query = "ip OR port";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(
+        result,
+        "Block with 'ip' should match the OR query even without 'port'"
+    );
+
+    // Test with a more complex query
+    let query = "(ip OR port) AND config";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(
+        result,
+        "Block with 'ip' and 'config' should match the complex query"
+    );
+}
+
+#[test]
+fn test_filter_tokenized_block_with_excluded_terms() {
+    // Create a tokenized content with some terms
+    let tokenized_content = vec![
+        "ip".to_string(),
+        "whitelist".to_string(),
+        "config".to_string(),
+    ];
+
+    // Test with a query that has excluded terms
+    let query = "ip -test";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(result, "Block with 'ip' but without 'test' should match");
+
+    // Create a tokenized content with the excluded term
+    let tokenized_content_with_excluded = vec![
+        "ip".to_string(),
+        "whitelist".to_string(),
+        "test".to_string(),
+    ];
+
+    // Test filtering with the excluded term present
+    let result = filter_tokenized_block(
+        &tokenized_content_with_excluded,
+        &plan.term_indices,
+        &plan,
+        true,
+    );
+    assert!(
+        !result,
+        "Block with 'ip' and 'test' should NOT match because 'test' is excluded"
+    );
+}
+
+#[test]
+fn test_filter_tokenized_block_with_complex_queries() {
+    // Create a tokenized content
+    let tokenized_content = vec![
+        "ip".to_string(),
+        "whitelist".to_string(),
+        "config".to_string(),
+        "server".to_string(),
+    ];
+
+    // Test with a complex query with AND, OR, and excluded terms
+    let query = "(ip OR port) AND (whitelist OR config) -test";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(
+        result,
+        "Block with 'ip', 'whitelist', and 'config' but without 'test' should match"
+    );
+
+    // Test with a different tokenized content that doesn't match
+    let tokenized_content_missing_required = vec!["port".to_string(), "server".to_string()];
+
+    // Test filtering
+    let result = filter_tokenized_block(
+        &tokenized_content_missing_required,
+        &plan.term_indices,
+        &plan,
+        true,
+    );
+    assert!(
+        !result,
+        "Block with only 'port' and 'server' should NOT match because it's missing 'whitelist' or 'config'"
+    );
+}
+
+#[test]
+fn test_filter_tokenized_block_empty_content() {
+    // Create an empty tokenized content
+    let tokenized_content: Vec<String> = vec![];
+
+    // Test with a simple query
+    let query = "ip AND whitelist";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test filtering
+    let result = filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+    assert!(!result, "Empty block should not match any query");
+}
+
+#[test]
+fn test_filter_tokenized_block_comparison_with_line_based() {
+    // This test compares the behavior of filter_tokenized_block with filter_code_block_with_ast
+    // to ensure they produce the same results for equivalent inputs
+
+    // Create a tokenized content
+    let tokenized_content = vec![
+        "ip".to_string(),
+        "whitelist".to_string(),
+        "config".to_string(),
+    ];
+
+    // Create a query
+    let query = "ip AND whitelist";
+    let plan = create_query_plan(query, false).unwrap();
+
+    // Test tokenized filtering
+    let tokenized_result =
+        filter_tokenized_block(&tokenized_content, &plan.term_indices, &plan, true);
+
+    // Import the line-based filtering function for comparison
+    use probe_code::search::file_processing::filter_code_block_with_ast;
+
+    // Create equivalent line-based term matches
+    let mut term_matches = HashMap::new();
+    term_matches.insert(0, HashSet::from([1])); // ip on line 1
+    term_matches.insert(1, HashSet::from([1])); // whitelist on line 1
+    term_matches.insert(2, HashSet::from([1])); // config on line 1
+
+    // Test line-based filtering
+    let line_based_result = filter_code_block_with_ast((1, 1), &term_matches, &plan, true);
+
+    // Compare results
+    assert_eq!(
+        tokenized_result, line_based_result,
+        "Tokenized filtering and line-based filtering should produce the same result"
+    );
+}