clean-code-tools 1.0.1

package/src/python/mcp_server/markdown.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from mcp_server.models import MarkdownSection
+from mcp_server.text import approximate_tokens, detected_record_id, slugless
+from mcp_server.utils.unique_strings import unique_strings
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+CODE_FENCE_RE = re.compile(r"^\s*```")
+MAX_SECTION_TOKENS = 1_200
+TARGET_SECTION_TOKENS = 850
+
+
+def markdown_sections(path: Path, *, root: Path) -> list[MarkdownSection]:
+    # pylint: disable=too-many-locals
+    relative_path = path.relative_to(root).as_posix()
+    lines = path.read_text().splitlines()
+    sections: list[MarkdownSection] = []
+    stack: list[str] = []
+    current_heading = path.stem
+    current_path = (path.stem,)
+    current_start = 1
+    current_body: list[str] = []
+    in_code = False
+
+    def flush(end_line: int) -> None:
+        body = "\n".join(current_body).strip()
+        if body:
+            sections.append(
+                MarkdownSection(
+                    source_file=relative_path,
+                    section_path=tuple(current_path),
+                    heading=current_heading,
+                    body=body,
+                    start_line=current_start,
+                    end_line=end_line,
+                )
+            )
+
+    for line_number, line in enumerate(lines, start=1):
+        if CODE_FENCE_RE.match(line):
+            in_code = not in_code
+            current_body.append(line)
+            continue
+        heading = HEADING_RE.match(line) if not in_code else None
+        if heading:
+            flush(line_number - 1)
+            level = len(heading.group(1))
+            text = heading.group(2).strip()
+            stack = stack[: level - 1]
+            stack.append(text)
+            current_heading = text
+            current_path = tuple(stack)
+            current_start = line_number
+            current_body = [line]
+            continue
+        current_body.append(line)
+
+    flush(len(lines))
+    return sections
+
+
+def split_section_body(body: str) -> list[str]:
+    if approximate_tokens(body) <= MAX_SECTION_TOKENS:
+        return [body]
+    blocks = semantic_blocks(body)
+    chunks: list[str] = []
+    current: list[str] = []
+    current_tokens = 0
+    for block in blocks:
+        block_tokens = approximate_tokens(block)
+        if current and current_tokens + block_tokens > TARGET_SECTION_TOKENS:
+            chunks.append("\n\n".join(current).strip())
+            current = []
+            current_tokens = 0
+        current.append(block)
+        current_tokens += block_tokens
+    if current:
+        chunks.append("\n\n".join(current).strip())
+    return [chunk for chunk in chunks if chunk]
+
+
+def semantic_blocks(body: str) -> list[str]:
+    blocks: list[str] = []
+    current: list[str] = []
+    in_code = False
+    for line in body.splitlines():
+        if CODE_FENCE_RE.match(line):
+            in_code = not in_code
+            current.append(line)
+            continue
+        if not in_code and not line.strip():
+            if current:
+                blocks.append("\n".join(current).strip())
+                current = []
+            continue
+        current.append(line)
+    if current:
+        blocks.append("\n".join(current).strip())
+    return blocks
+
+
+def infer_markdown_rule_family(section: MarkdownSection) -> str:
+    text = " ".join(section.section_path).lower()
+    if "eslint" in text or "ruff" in text or "lint" in text:
+        return "linting"
+    if "comment" in text:
+        return "comments"
+    if "test" in text:
+        return "tests"
+    if "function" in text or "argument" in text:
+        return "functions"
+    if "name" in text:
+        return "naming"
+    return "documentation"
+
+
+def markdown_aliases(section: MarkdownSection) -> tuple[str, ...]:
+    aliases = [section.heading, *section.section_path]
+    record_id = detected_record_id(section.heading)
+    if record_id:
+        aliases.append(record_id)
+    return tuple(unique_strings([slugless(alias) for alias in aliases if alias.strip()]))

package/src/python/mcp_server/models.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+CHUNKER_VERSION = "clean-code-semantic-v1"
+DEFAULT_EMBEDDING_MODEL = "BAAI/bge-small-en-v1.5"
+DEFAULT_EMBEDDING_PROVIDER = "fastembed/cpu"
+
+JsonDict = dict[str, Any]
+Confidence = Literal["high", "medium", "low"]
+
+
+@dataclass(frozen=True, slots=True)
+class CleanCodeChunk:  # pylint: disable=too-many-instance-attributes
+    chunk_id: str
+    object_id: str
+    source_file: str
+    source_kind: str
+    record_id: str
+    title: str
+    topic: str
+    section_path: tuple[str, ...]
+    chunk_kind: str
+    chunk_index: int
+    rule_family: str
+    lintability: str
+    aliases: tuple[str, ...]
+    languages: tuple[str, ...]
+    lint_candidates: tuple[str, ...]
+    content_text: str
+    embedding_text: str
+    display_text: str
+    text_hash: str
+
+    @property
+    def properties(self) -> JsonDict:
+        return {
+            "chunkId": self.chunk_id,
+            "sourceFile": self.source_file,
+            "sourceKind": self.source_kind,
+            "recordId": self.record_id,
+            "title": self.title,
+            "topic": self.topic,
+            "sectionPath": list(self.section_path),
+            "chunkKind": self.chunk_kind,
+            "chunkIndex": self.chunk_index,
+            "ruleFamily": self.rule_family,
+            "lintability": self.lintability,
+            "aliases": list(self.aliases),
+            "languages": list(self.languages),
+            "lintCandidates": list(self.lint_candidates),
+            "contentText": self.content_text,
+            "embeddingText": self.embedding_text,
+            "displayText": self.display_text,
+            "textHash": self.text_hash,
+            "chunkerVersion": CHUNKER_VERSION,
+            "embeddingModel": DEFAULT_EMBEDDING_MODEL,
+            "embeddingProvider": DEFAULT_EMBEDDING_PROVIDER,
+            "createdAt": datetime.now(UTC).isoformat(),
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class MarkdownSection:
+    source_file: str
+    section_path: tuple[str, ...]
+    heading: str
+    body: str
+    start_line: int
+    end_line: int

package/src/python/mcp_server/ranking.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+from mcp_server.corpus import build_chunks
+from mcp_server.models import DEFAULT_EMBEDDING_MODEL, JsonDict
+from mcp_server.ranking_scoring import (
+    dedupe_pattern_results,
+    distance_for,
+    row_matches_filters,
+    score_pattern_row,
+)
+from mcp_server.text import (
+    lexical_score,
+    query_tokens,
+)
+from mcp_server.weaviate import COLLECTION_NAME, search_chunks
+
+
+def search_pattern_records(  # noqa: PLR0913  # pylint: disable=too-many-arguments,too-many-locals
+    *,
+    query: str,
+    url: str,
+    collection_name: str = COLLECTION_NAME,
+    model_name: str = DEFAULT_EMBEDDING_MODEL,
+    limit: int = 8,
+    language: str = "any",
+    rule_families: tuple[str, ...] = (),
+    topics: tuple[str, ...] = (),
+    lintability: tuple[str, ...] = (),
+    source_kinds: tuple[str, ...] = ("clean_code_pattern",),
+) -> JsonDict:
+    vector_limit = max(limit * 4, 25)
+    vector_rows = search_chunks(
+        query=query,
+        url=url,
+        collection_name=collection_name,
+        model_name=model_name,
+        limit=vector_limit,
+    )
+    return rank_pattern_rows(
+        query=query,
+        vector_rows=vector_rows,
+        limit=limit,
+        language=language,
+        rule_families=rule_families,
+        topics=topics,
+        lintability=lintability,
+        source_kinds=source_kinds,
+    )
+
+
+def rank_pattern_rows(  # noqa: PLR0913  # pylint: disable=too-many-arguments,too-many-locals
+    *,
+    query: str,
+    vector_rows: list[JsonDict],
+    limit: int = 8,
+    language: str = "any",
+    rule_families: tuple[str, ...] = (),
+    topics: tuple[str, ...] = (),
+    lintability: tuple[str, ...] = (),
+    source_kinds: tuple[str, ...] = ("clean_code_pattern",),
+) -> JsonDict:
+    rows_by_chunk_id: dict[str, JsonDict] = {}
+    for row in [*vector_rows, *local_keyword_rows(query)]:
+        chunk_id = str(row.get("chunkId", ""))
+        if chunk_id:
+            rows_by_chunk_id[chunk_id] = merge_search_rows(rows_by_chunk_id.get(chunk_id), row)
+
+    filters = {
+        "language": language,
+        "rule_families": rule_families,
+        "topics": topics,
+        "lintability": lintability,
+        "source_kinds": source_kinds,
+    }
+    ranked: list[JsonDict] = []
+    for row in rows_by_chunk_id.values():
+        if not row_matches_filters(row, filters):
+            continue
+        ranked.append(score_pattern_row(row, query=query, filters=filters))
+
+    ranked.sort(key=lambda item: float(item["score"]), reverse=True)
+    deduped = dedupe_pattern_results(ranked)
+    results = deduped[:limit]
+    return {
+        "query": query,
+        "filters_applied": filters,
+        "results": results,
+        "no_strong_match": not results or results[0]["confidence"] != "high",
+    }
+
+
+def local_keyword_rows(query: str) -> list[JsonDict]:
+    query_terms = set(query_tokens(query))
+    if not query_terms:
+        return []
+    matches: list[JsonDict] = []
+    for chunk in build_chunks():
+        haystack = " ".join(
+            (
+                chunk.record_id,
+                chunk.title,
+                chunk.topic,
+                chunk.rule_family,
+                chunk.lintability,
+                " ".join(chunk.aliases),
+                " ".join(chunk.lint_candidates),
+                chunk.content_text[:600],
+            )
+        )
+        if lexical_score(query_terms, haystack) > 0:
+            matches.append(chunk.properties | {"_additional": {"id": chunk.object_id}})
+    return matches
+
+
+def merge_search_rows(existing: JsonDict | None, row: JsonDict) -> JsonDict:
+    if existing is None:
+        return row
+    existing_distance = distance_for(existing)
+    row_distance = distance_for(row)
+    if existing_distance is None:
+        return row
+    if row_distance is None:
+        return existing
+    return row if row_distance < existing_distance else existing

package/src/python/mcp_server/ranking_scoring.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+from mcp_server.models import Confidence, JsonDict
+from mcp_server.text import (
+    lexical_score,
+    query_tokens,
+    searchable_row_text,
+    semantic_similarity,
+)
+from mcp_server.utils.unique_strings import unique_strings
+
+CONSERVATIVE_CONTEXT_THRESHOLD = 0.80
+HIGH_CONFIDENCE_THRESHOLD = 0.72
+EXACT_HIGH_CONFIDENCE_THRESHOLD = 0.40
+MEDIUM_CONFIDENCE_THRESHOLD = 0.45
+EXACT_MATCH_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "be",
+    "by",
+    "code",
+    "do",
+    "for",
+    "in",
+    "is",
+    "it",
+    "of",
+    "on",
+    "or",
+    "the",
+    "thi",
+    "this",
+    "to",
+    "when",
+    "with",
+}
+NON_SPECIFIC_EXACT_ALIASES = {
+    "clean code",
+    "code smell",
+    "comments",
+    "error handling",
+    "function smell",
+    "functions",
+    "function design",
+    "planning guidance",
+    "refactoring rule",
+}
+BROAD_CATEGORY_QUERY_TERMS = {
+    "clean",
+    "code",
+    "comment",
+    "comments",
+    "error",
+    "function",
+    "handling",
+    "guidance",
+    "planning",
+    "python",
+    "refactoring",
+    "rule",
+    "smell",
+    "typescript",
+}
+VAGUE_QUERY_TERMS = {"stuff", "thing", "things"}
+
+
+def row_matches_filters(row: JsonDict, filters: JsonDict) -> bool:
+    source_kinds = tuple(str(value) for value in filters["source_kinds"])
+    if source_kinds and str(row.get("sourceKind", "")) not in source_kinds:
+        return False
+
+    language = str(filters["language"])
+    languages = tuple(str(value) for value in row.get("languages", []))
+    if language not in {"", "any", "both"} and language not in languages:
+        return False
+    if language == "both" and not {"typescript", "python"} <= set(languages):
+        return False
+
+    rule_families = {str(value) for value in filters["rule_families"]}
+    if rule_families and str(row.get("ruleFamily", "")) not in rule_families:
+        return False
+
+    topics = {str(value) for value in filters["topics"]}
+    if topics and str(row.get("topic", "")) not in topics:
+        return False
+
+    lintability = {str(value) for value in filters["lintability"]}
+    return not lintability or str(row.get("lintability", "")) in lintability
+
+
+def score_pattern_row(row: JsonDict, *, query: str, filters: JsonDict) -> JsonDict:
+    query_terms = set(query_tokens(query))
+    distance = distance_for(row)
+    semantic_score = semantic_similarity(distance)
+    keyword_score = lexical_score(query_terms, searchable_row_text(row))
+    metadata_boost, match_reasons = metadata_boost_and_reasons(row, filters, query_terms)
+    context_penalty, context_reasons = context_penalty_and_reasons(query_terms)
+    match_reasons.extend(context_reasons)
+    score = max(0.0, min(1.0, semantic_score * 0.60 + keyword_score * 0.25 + metadata_boost - context_penalty))
+    confidence = confidence_for(score, match_reasons, semantic_score=semantic_score)
+    return {
+        "pattern_id": row.get("recordId", ""),
+        "chunk_id": row.get("chunkId", ""),
+        "title": row.get("title", ""),
+        "topic": row.get("topic", ""),
+        "rule_family": row.get("ruleFamily", ""),
+        "lintability": row.get("lintability", ""),
+        "languages": row.get("languages", []),
+        "aliases": row.get("aliases", []),
+        "lint_candidates": row.get("lintCandidates", []),
+        "source_kind": row.get("sourceKind", ""),
+        "chunk_kind": row.get("chunkKind", ""),
+        "section_path": row.get("sectionPath", []),
+        "score": round(score, 4),
+        "confidence": confidence,
+        "distance": distance,
+        "match_reasons": match_reasons,
+        "snippet": " ".join(str(row.get("contentText", "")).split())[:500],
+        "source": {
+            "source_file": row.get("sourceFile", ""),
+            "source_kind": row.get("sourceKind", ""),
+            "text_hash": row.get("textHash", ""),
+        },
+        "score_breakdown": {
+            "semantic_score": round(semantic_score, 4),
+            "keyword_score": round(keyword_score, 4),
+            "metadata_boost": round(metadata_boost, 4),
+            "context_penalty": round(context_penalty, 4),
+        },
+    }
+
+
+def dedupe_pattern_results(rows: list[JsonDict]) -> list[JsonDict]:
+    by_record_id: dict[str, JsonDict] = {}
+    for row in rows:
+        record_id = str(row.get("pattern_id") or row.get("chunk_id"))
+        if record_id not in by_record_id or float(row["score"]) > float(by_record_id[record_id]["score"]):
+            by_record_id[record_id] = row
+    return sorted(by_record_id.values(), key=lambda item: float(item["score"]), reverse=True)
+
+
+def metadata_boost_and_reasons(
+    row: JsonDict,
+    filters: JsonDict,
+    query_terms: set[str],
+) -> tuple[float, list[str]]:
+    boost = 0.0
+    reasons: list[str] = []
+    source_kind = str(row.get("sourceKind", ""))
+    if source_kind == "clean_code_pattern":
+        boost += 0.05
+        reasons.append("canonical clean-code pattern")
+
+    language = str(filters["language"])
+    languages = {str(value) for value in row.get("languages", [])}
+    if language not in {"", "any"} and (language == "both" or language in languages):
+        boost += 0.05
+        reasons.append(f"language matched: {language}")
+
+    lintability = str(row.get("lintability", ""))
+    if lintability and lintability in set(filters["lintability"]):
+        boost += 0.05
+        reasons.append(f"lintability matched: {lintability}")
+
+    exact_terms = exact_match_terms(row, query_terms)
+    if exact_terms:
+        boost += 0.15
+        reasons.append(f"matched exact terms: {', '.join(exact_terms[:4])}")
+
+    if not reasons:
+        reasons.append("semantic similarity")
+    return min(boost, 0.30), reasons
+
+
+def context_penalty_and_reasons(query_terms: set[str]) -> tuple[float, list[str]]:
+    safe_context_terms = {"generated", "fixture", "fixtures", "test", "tests"}
+    matched = sorted(query_terms & safe_context_terms)
+    reasons: list[str] = []
+    if matched:
+        reasons.append(f"conservative context: {', '.join(matched)}")
+    meaningful_terms = query_terms - EXACT_MATCH_STOPWORDS
+    if meaningful_terms and meaningful_terms <= BROAD_CATEGORY_QUERY_TERMS:
+        reasons.append("conservative context: broad category query")
+    if meaningful_terms and meaningful_terms <= VAGUE_QUERY_TERMS:
+        reasons.append("conservative context: vague query")
+    if {"todo", "tracked", "issue"} <= query_terms:
+        reasons.append("conservative context: compliant tracked TODO")
+    if not reasons:
+        return 0.0, []
+    return 0.35, reasons
+
+
+def exact_match_terms(row: JsonDict, query_terms: set[str]) -> list[str]:
+    values = [
+        str(row.get("recordId", "")),
+        str(row.get("title", "")),
+        *[str(value) for value in row.get("aliases", [])],
+        *[str(value) for value in row.get("lintCandidates", [])],
+    ]
+    matched: list[str] = []
+    for value in values:
+        normalized_value = " ".join(query_tokens(value))
+        if normalized_value in NON_SPECIFIC_EXACT_ALIASES:
+            continue
+        terms = set(query_tokens(value))
+        meaningful_terms = terms - EXACT_MATCH_STOPWORDS
+        if terms and meaningful_terms and terms <= query_terms:
+            matched.append(value)
+    return unique_strings(matched)
+
+
+def distance_for(row: JsonDict) -> float | None:
+    additional = row.get("_additional")
+    if not isinstance(additional, dict):
+        return None
+    distance = additional.get("distance")
+    return float(distance) if isinstance(distance, int | float) else None
+
+
+def confidence_for(score: float, match_reasons: list[str], *, semantic_score: float) -> Confidence:
+    if any(reason.startswith("conservative context") for reason in match_reasons):
+        return "low" if score < CONSERVATIVE_CONTEXT_THRESHOLD else "medium"
+    has_exact = any(reason.startswith("matched exact terms") for reason in match_reasons)
+    if score >= HIGH_CONFIDENCE_THRESHOLD or (
+        has_exact and semantic_score > 0 and score >= EXACT_HIGH_CONFIDENCE_THRESHOLD
+    ):
+        return "high"
+    if score >= MEDIUM_CONFIDENCE_THRESHOLD:
+        return "medium"
+    return "low"