claude-code-workflow 6.3.19 → 6.3.21

package/ccw/src/templates/dashboard-js/views/codexlens-manager.js

@@ -1109,6 +1109,16 @@ var ENV_VAR_GROUPS = {
       'CODEXLENS_CASCADE_COARSE_K': { labelKey: 'codexlens.envField.coarseK', type: 'number', placeholder: '100', default: '100', settingsPath: 'cascade.coarse_k', min: 10, max: 500 },
       'CODEXLENS_CASCADE_FINE_K': { labelKey: 'codexlens.envField.fineK', type: 'number', placeholder: '10', default: '10', settingsPath: 'cascade.fine_k', min: 1, max: 100 }
     }
+  },
+  chunking: {
+    labelKey: 'codexlens.envGroup.chunking',
+    icon: 'scissors',
+    vars: {
+      'CHUNK_STRIP_COMMENTS': { labelKey: 'codexlens.envField.stripComments', type: 'select', options: ['true', 'false'], default: 'true', settingsPath: 'chunking.strip_comments' },
+      'CHUNK_STRIP_DOCSTRINGS': { labelKey: 'codexlens.envField.stripDocstrings', type: 'select', options: ['true', 'false'], default: 'true', settingsPath: 'chunking.strip_docstrings' },
+      'RERANKER_TEST_FILE_PENALTY': { labelKey: 'codexlens.envField.testFilePenalty', type: 'number', placeholder: '0.0', default: '0.0', settingsPath: 'reranker.test_file_penalty', min: 0, max: 1, step: 0.1 },
+      'RERANKER_DOCSTRING_WEIGHT': { labelKey: 'codexlens.envField.docstringWeight', type: 'number', placeholder: '1.0', default: '1.0', settingsPath: 'reranker.docstring_weight', min: 0, max: 1, step: 0.1 }
+    }
   }
 };
 
@@ -3603,7 +3613,7 @@ async function initCodexLensIndex(indexType, embeddingModel, embeddingBackend, m
           // Install semantic dependencies first
           showRefreshToast(t('codexlens.installingDeps') || 'Installing semantic dependencies...', 'info');
           try {
-            var installResponse = await fetch('/api/codexlens/semantic/install', { method: 'POST' });
+            var installResponse = await csrfFetch('/api/codexlens/semantic/install', { method: 'POST' });
             var installResult = await installResponse.json();
 
             if (!installResult.success) {
@@ -5373,7 +5383,7 @@ function initCodexLensManagerPageEvents(currentConfig) {
       saveBtn.disabled = true;
       saveBtn.innerHTML = '<span class="animate-pulse">' + t('common.saving') + '</span>';
       try {
-        var response = await fetch('/api/codexlens/config', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ index_dir: newIndexDir }) });
+        var response = await csrfFetch('/api/codexlens/config', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ index_dir: newIndexDir }) });
         var result = await response.json();
         if (result.success) { showRefreshToast(t('codexlens.configSaved'), 'success'); renderCodexLensManager(); }
         else { showRefreshToast(t('common.saveFailed') + ': ' + result.error, 'error'); }

package/ccw/src/templates/dashboard-js/views/memory.js

@@ -1114,7 +1114,7 @@ async function deleteInsight(insightId) {
   if (!confirm(t('memory.confirmDeleteInsight'))) return;
 
   try {
-    var response = await fetch('/api/memory/insights/' + insightId, { method: 'DELETE' });
+    var response = await csrfFetch('/api/memory/insights/' + insightId, { method: 'DELETE' });
     if (!response.ok) throw new Error('Failed to delete insight');
 
     selectedInsight = null;

package/ccw/src/templates/dashboard-js/views/prompt-history.js

@@ -431,7 +431,7 @@ async function deletePromptInsight(insightId) {
   if (!confirm(isZh() ? '确定要删除这条洞察记录吗？' : 'Are you sure you want to delete this insight?')) return;
 
   try {
-    var response = await fetch('/api/memory/insights/' + insightId, { method: 'DELETE' });
+    var response = await csrfFetch('/api/memory/insights/' + insightId, { method: 'DELETE' });
     if (!response.ok) throw new Error('Failed to delete insight');
 
     selectedPromptInsight = null;

package/codex-lens/src/codexlens/cli/embedding_manager.py

@@ -535,10 +535,15 @@ def generate_embeddings(
 
         # skip_token_count=True: Use fast estimation (len/4) instead of expensive tiktoken
         # This significantly reduces CPU usage with minimal impact on metadata accuracy
+        # Load chunk stripping config from settings
+        from codexlens.config import Config
+        chunk_cfg = Config.load()
         chunker = Chunker(config=ChunkConfig(
             max_chunk_size=chunk_size,
             overlap=overlap,
-            skip_token_count=True
+            skip_token_count=True,
+            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
+            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
         ))
 
         # Log embedder info with endpoint count for multi-endpoint mode
@@ -1307,10 +1312,15 @@ def generate_dense_embeddings_centralized(
                 "error": f"Invalid embedding backend: {embedding_backend}",
             }
 
+        # Load chunk stripping config from settings
+        from codexlens.config import Config
+        chunk_cfg = Config.load()
         chunker = Chunker(config=ChunkConfig(
             max_chunk_size=chunk_size,
             overlap=overlap,
-            skip_token_count=True
+            skip_token_count=True,
+            strip_comments=getattr(chunk_cfg, 'chunk_strip_comments', True),
+            strip_docstrings=getattr(chunk_cfg, 'chunk_strip_docstrings', True),
         ))
 
         if progress_callback:
@@ -1319,8 +1329,7 @@ def generate_dense_embeddings_centralized(
             progress_callback(f"Using model: {embedder.model_name} ({embedder.embedding_dim} dimensions)")
 
         # Calculate dynamic batch size based on model capacity
-        from codexlens.config import Config
-        batch_config = Config.load()
+        batch_config = chunk_cfg  # Reuse already loaded config
         effective_batch_size = calculate_dynamic_batch_size(batch_config, embedder)
 
         if progress_callback and batch_config.api_batch_size_dynamic:

package/codex-lens/src/codexlens/config.py

@@ -141,6 +141,12 @@ class Config:
     reranker_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"
     reranker_top_k: int = 50
     reranker_max_input_tokens: int = 8192  # Maximum tokens for reranker API batching
+    reranker_chunk_type_weights: Optional[Dict[str, float]] = None  # Weights for chunk types: {"code": 1.0, "docstring": 0.7}
+    reranker_test_file_penalty: float = 0.0  # Penalty for test files (0.0-1.0, e.g., 0.2 = 20% reduction)
+
+    # Chunk stripping configuration (for semantic embedding)
+    chunk_strip_comments: bool = True  # Strip comments from code chunks
+    chunk_strip_docstrings: bool = True  # Strip docstrings from code chunks
 
     # Cascade search configuration (two-stage retrieval)
     enable_cascade_search: bool = False  # Enable cascade search (coarse + fine ranking)
@@ -545,6 +551,35 @@ class Config:
             except ValueError:
                 log.warning("Invalid RERANKER_MAX_INPUT_TOKENS in .env: %r", reranker_max_tokens)
 
+        # Reranker tuning from environment
+        test_penalty = get_env("RERANKER_TEST_FILE_PENALTY")
+        if test_penalty:
+            try:
+                self.reranker_test_file_penalty = float(test_penalty)
+                log.debug("Overriding reranker_test_file_penalty from .env: %s", self.reranker_test_file_penalty)
+            except ValueError:
+                log.warning("Invalid RERANKER_TEST_FILE_PENALTY in .env: %r", test_penalty)
+
+        docstring_weight = get_env("RERANKER_DOCSTRING_WEIGHT")
+        if docstring_weight:
+            try:
+                weight = float(docstring_weight)
+                self.reranker_chunk_type_weights = {"code": 1.0, "docstring": weight}
+                log.debug("Overriding reranker docstring weight from .env: %s", weight)
+            except ValueError:
+                log.warning("Invalid RERANKER_DOCSTRING_WEIGHT in .env: %r", docstring_weight)
+
+        # Chunk stripping from environment
+        strip_comments = get_env("CHUNK_STRIP_COMMENTS")
+        if strip_comments:
+            self.chunk_strip_comments = strip_comments.lower() in ("true", "1", "yes")
+            log.debug("Overriding chunk_strip_comments from .env: %s", self.chunk_strip_comments)
+
+        strip_docstrings = get_env("CHUNK_STRIP_DOCSTRINGS")
+        if strip_docstrings:
+            self.chunk_strip_docstrings = strip_docstrings.lower() in ("true", "1", "yes")
+            log.debug("Overriding chunk_strip_docstrings from .env: %s", self.chunk_strip_docstrings)
+
     @classmethod
     def load(cls) -> "Config":
         """Load config with settings from file."""

package/codex-lens/src/codexlens/env_config.py

@@ -45,6 +45,12 @@ ENV_VARS = {
     # General configuration
     "CODEXLENS_DATA_DIR": "Custom data directory path",
     "CODEXLENS_DEBUG": "Enable debug mode (true/false)",
+    # Chunking configuration
+    "CHUNK_STRIP_COMMENTS": "Strip comments from code chunks for embedding: true/false (default: true)",
+    "CHUNK_STRIP_DOCSTRINGS": "Strip docstrings from code chunks for embedding: true/false (default: true)",
+    # Reranker tuning
+    "RERANKER_TEST_FILE_PENALTY": "Penalty for test files in reranking: 0.0-1.0 (default: 0.0)",
+    "RERANKER_DOCSTRING_WEIGHT": "Weight for docstring chunks in reranking: 0.0-1.0 (default: 1.0)",
 }
 
 

package/codex-lens/src/codexlens/search/chain_search.py

@@ -1816,12 +1816,22 @@ class ChainSearchEngine:
         # Use cross_encoder_rerank from ranking module
         from codexlens.search.ranking import cross_encoder_rerank
 
+        # Get chunk_type weights and test_file_penalty from config
+        chunk_type_weights = None
+        test_file_penalty = 0.0
+
+        if self._config is not None:
+            chunk_type_weights = getattr(self._config, "reranker_chunk_type_weights", None)
+            test_file_penalty = getattr(self._config, "reranker_test_file_penalty", 0.0)
+
         return cross_encoder_rerank(
             query=query,
             results=results,
             reranker=reranker,
             top_k=top_k,
             batch_size=32,
+            chunk_type_weights=chunk_type_weights,
+            test_file_penalty=test_file_penalty,
         )
 
     def search_files_only(self, query: str,

package/codex-lens/src/codexlens/search/ranking.py

@@ -613,11 +613,24 @@ def cross_encoder_rerank(
     reranker: Any,
     top_k: int = 50,
     batch_size: int = 32,
+    chunk_type_weights: Optional[Dict[str, float]] = None,
+    test_file_penalty: float = 0.0,
 ) -> List[SearchResult]:
     """Second-stage reranking using a cross-encoder model.
 
     This function is dependency-agnostic: callers can pass any object that exposes
     a compatible `score_pairs(pairs, batch_size=...)` method.
+
+    Args:
+        query: Search query string
+        results: List of search results to rerank
+        reranker: Cross-encoder model with score_pairs or predict method
+        top_k: Number of top results to rerank
+        batch_size: Batch size for reranking
+        chunk_type_weights: Optional weights for different chunk types.
+            Example: {"code": 1.0, "docstring": 0.7} - reduce docstring influence
+        test_file_penalty: Penalty applied to test files (0.0-1.0).
+            Example: 0.2 means test files get 20% score reduction
     """
     if not results:
         return []
@@ -667,13 +680,50 @@ def cross_encoder_rerank(
 
     reranked_results: List[SearchResult] = []
 
+    # Helper to detect test files
+    def is_test_file(path: str) -> bool:
+        if not path:
+            return False
+        basename = path.split("/")[-1].split("\\")[-1]
+        return (
+            basename.startswith("test_") or
+            basename.endswith("_test.py") or
+            basename.endswith(".test.ts") or
+            basename.endswith(".test.js") or
+            basename.endswith(".spec.ts") or
+            basename.endswith(".spec.js") or
+            "/tests/" in path or
+            "\\tests\\" in path or
+            "/test/" in path or
+            "\\test\\" in path
+        )
+
     for idx, result in enumerate(results):
         if idx < rerank_count:
             prev_score = float(result.score)
             ce_score = scores[idx]
             ce_prob = probs[idx]
+
+            # Base combined score
             combined_score = 0.5 * prev_score + 0.5 * ce_prob
 
+            # Apply chunk_type weight adjustment
+            if chunk_type_weights:
+                chunk_type = None
+                if result.chunk and hasattr(result.chunk, "metadata"):
+                    chunk_type = result.chunk.metadata.get("chunk_type")
+                elif result.metadata:
+                    chunk_type = result.metadata.get("chunk_type")
+
+                if chunk_type and chunk_type in chunk_type_weights:
+                    weight = chunk_type_weights[chunk_type]
+                    # Apply weight to CE contribution only
+                    combined_score = 0.5 * prev_score + 0.5 * ce_prob * weight
+
+            # Apply test file penalty
+            if test_file_penalty > 0 and is_test_file(result.path):
+                combined_score = combined_score * (1.0 - test_file_penalty)
+
             reranked_results.append(
                 SearchResult(
                     path=result.path,