@theglitchking/babel-fish 2.0.3 → 2.4.0

package/.claude/project-map/generate.py

@@ -13,10 +13,11 @@ import ast
 import argparse
 import hashlib
 import json
+from fnmatch import fnmatch
 import re
 import subprocess
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
@@ -33,6 +34,10 @@ MAP_DIR      = SCRIPT_DIR
 SECTIONS_DIR = MAP_DIR / "sections"
 CHECKSUMS    = MAP_DIR / "checksums.json"
 LEARNED_VOC  = MAP_DIR / "learned-vocabulary.json"
+GLOSSARY     = MAP_DIR / "glossary.json"
+# Bump when the glossary.json shape changes. Consumers should refuse a major
+# they do not recognise rather than guess.
+GLOSSARY_SCHEMA_VERSION = "1.0"
 
 # Resolve project root: two levels up from .claude/project-map/
 PROJECT_ROOT = MAP_DIR.parent.parent
@@ -53,13 +58,41 @@ def redact_secrets(text: str) -> str:
 # ── Checksum logic ───────────────────────────────────────────────────────────
 WATCHED_EXTENSIONS = {
     '.py', '.ts', '.tsx', '.js', '.jsx', '.go', '.java', '.kt',
-    '.yaml', '.yml', '.toml', '.json', '.prisma', '.sql', '.env',
+    '.yaml', '.yml', '.toml', '.json', '.prisma', '.sql', '.env', '.sh',
 }
 WATCHED_NAMES = {
     'docker-compose.yml', 'docker-compose.yaml', 'docker-compose.dev.yml',
     'package.json', 'requirements.txt', 'pyproject.toml', 'go.mod',
     'Cargo.toml', 'pom.xml', 'Gemfile', 'Makefile',
 }
+# Skill/command manifests — matched by path shape, not by extension, so
+# documentation .md files stay out of the watch set. fnmatch runs against the
+# repo-relative posix path, which anchors the glob at the root:
+# ".documentation/reference/commands/cli.md" does not match "commands/*.md".
+WATCHED_GLOBS = (
+    'skills/*/SKILL.md',
+    'commands/*.md',
+    # Dead until '.claude' leaves IGNORE_DIRS; kept as the anchor for when it does.
+    '.claude/skills/*/SKILL.md',
+    '.claude/commands/*.md',
+)
+
+# Directories build_doc_pointers_section() walks. Watched by PATH ONLY (no
+# mtime) so adding or removing a doc refreshes section 19 while editing one
+# does not churn the whole map.
+DOC_DIRS = ['docs', 'doc', 'documentation', 'wiki', '.docs', '.documentation']
+DOC_EXTS = {'.md', '.rst', '.txt', '.adoc'}
+# Auto-generated navigation, not documentation. A hit-em-with-the-docs tree
+# carries one INDEX.md + REGISTRY.md per domain (32 files for 15 domains), which
+# would otherwise crowd every real doc out of section 19's 30-entry cap.
+DOC_SKIP_NAMES = {'INDEX.md', 'REGISTRY.md'}
+# hewtd excludes archive/ from all of its own scans; deprecated docs are not
+# pointers worth handing an agent. 'reports' holds generated, timestamped audit
+# output — listing it is noise, and because every run writes a NEW filename it
+# would change the doc path set and force a full regeneration each time, which
+# is the churn the path-only doc hash exists to prevent.
+DOC_SKIP_DIRS = {'archive', 'reports'}
+
 IGNORE_DIRS = {
     '.git', 'node_modules', '__pycache__', '.venv', 'venv', 'env',
     'dist', 'build', '.next', '.nuxt', 'target', 'vendor', '.cache',
@@ -71,11 +104,34 @@ def collect_watched_files() -> list[Path]:
     for path in sorted(PROJECT_ROOT.rglob('*')):
         if any(p in IGNORE_DIRS for p in path.parts):
             continue
-        if path.is_file() and (path.suffix in WATCHED_EXTENSIONS or path.name in WATCHED_NAMES):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(PROJECT_ROOT).as_posix()
+        if (path.suffix in WATCHED_EXTENSIONS
+                or path.name in WATCHED_NAMES
+                or any(fnmatch(rel, g) for g in WATCHED_GLOBS)):
             result.append(path)
     return result
 
-def compute_checksum(files: list[Path]) -> str:
+
+def collect_doc_files() -> list[Path]:
+    """Docs section 19 points at. Hashed by path only — see compute_checksum."""
+    docs = []
+    for doc_dir in DOC_DIRS:
+        d = PROJECT_ROOT / doc_dir
+        if d.is_dir():
+            docs.extend(
+                f for f in d.rglob('*')
+                if f.is_file()
+                and f.suffix in DOC_EXTS
+                and f.name not in DOC_SKIP_NAMES
+                and not (DOC_SKIP_DIRS & set(f.relative_to(PROJECT_ROOT).parts))
+            )
+    # Root-level docs, matching build_doc_pointers_section()'s own glob.
+    docs.extend(f for f in PROJECT_ROOT.glob('*.md') if f.is_file())
+    return sorted(set(docs))
+
+def compute_checksum(files: list[Path], doc_files: list[Path] | None = None) -> str:
     h = hashlib.sha256()
     for f in files:
         h.update(str(f).encode())
@@ -83,6 +139,11 @@ def compute_checksum(files: list[Path]) -> str:
             h.update(str(f.stat().st_mtime_ns).encode())
         except OSError:
             pass
+    # ponytail: path only, no mtime — a new/renamed/deleted doc regenerates the
+    # section 19 pointer list, an edited one doesn't. Hashing doc mtimes would
+    # force a full regeneration on every prose edit.
+    for f in doc_files or []:
+        h.update(str(f).encode())
     return h.hexdigest()
 
 def load_checksums() -> dict:
@@ -710,9 +771,20 @@ class VocabularyBuilder:
         schemas: list[dict],
         features: list[dict],
         stack: dict,
+        skills: list[dict] | None = None,
     ) -> list[dict]:
         vocab: dict[str, dict] = {}
 
+        # From skills and slash commands. In a plugin repo this is the only
+        # source that fires — there are no routes or models to mine.
+        for sk in skills or []:
+            note = sk['description'][:120] if sk['description'] else f"{sk['kind']} manifest"
+            for alias in self._name_to_aliases(sk['name']):
+                self._add(vocab, alias, sk['kind'], sk['file'], note)
+            if sk['kind'] == 'command':
+                # humans say "/status" as often as "status"
+                self._add(vocab, f"/{sk['name']}", 'command', sk['file'], note)
+
         # From features
         for feat in features:
             name = feat['name']
@@ -781,6 +853,98 @@ class VocabularyBuilder:
         return {}
 
 
+# ── Skill / Command Manifest Parser ───────────────────────────────────────────
+
+class SkillParser:
+    """Claude skill and slash-command manifests.
+
+    In a plugin/skill repo these ARE the source: there are no routes or models
+    to extract, but a skill's frontmatter name + description is literally an
+    alias -> location pair, which is what section 01 wants.
+
+    The .claude/* patterns are globbed directly, the same way ToolsScanner
+    reaches .claude/skills, so a consuming project's installed skills are
+    picked up even though '.claude' is in IGNORE_DIRS. Those files are parsed
+    but not watched, so they refresh on the next regeneration rather than
+    immediately — see WATCHED_GLOBS.
+    """
+
+    PATTERNS = (
+        ('skills/*/SKILL.md', 'skill'),
+        ('.claude/skills/*/SKILL.md', 'skill'),
+        ('commands/*.md', 'command'),
+        ('.claude/commands/*.md', 'command'),
+    )
+
+    def parse(self) -> list[dict]:
+        found: dict[str, dict] = {}
+        for pattern, kind in self.PATTERNS:
+            for f in sorted(PROJECT_ROOT.glob(pattern)):
+                meta = self._frontmatter(f)
+                if meta is None:
+                    continue
+                # commands carry no 'name:' — the filename is the command.
+                name = str(meta.get('name') or '').strip()
+                if not name:
+                    name = f.parent.name if f.name == 'SKILL.md' else f.stem
+                desc = ' '.join(str(meta.get('description') or '').split())
+                rel = str(f.relative_to(PROJECT_ROOT))
+                found.setdefault(rel, {
+                    'name': name,
+                    'kind': kind,
+                    'file': rel,
+                    'description': desc,
+                })
+        return list(found.values())
+
+    def _frontmatter(self, f: Path) -> dict | None:
+        """Leading --- ... --- YAML block, or None when absent/unparseable."""
+        try:
+            text = f.read_text(encoding='utf-8', errors='replace')
+        except OSError:
+            return None
+        if not text.startswith('---'):
+            return None
+        end = text.find('\n---', 3)
+        if end == -1:
+            return None
+        block = text[3:end]
+        if HAS_YAML:
+            try:
+                data = yaml.safe_load(block)
+                if isinstance(data, dict):
+                    return data
+            except Exception:
+                pass
+        return self._frontmatter_regex(block)
+
+    def _frontmatter_regex(self, block: str) -> dict:
+        """pyyaml-free fallback. Handles 'key: value' and 'key: |' blocks —
+        enough for name/description, which is all this parser reads."""
+        out: dict[str, str] = {}
+        lines = block.splitlines()
+        i = 0
+        while i < len(lines):
+            m = re.match(r'^([A-Za-z_][\w-]*):\s*(.*)$', lines[i])
+            if not m:
+                i += 1
+                continue
+            key, val = m.group(1), m.group(2).strip()
+            if val in ('|', '>', '|-', '>-', ''):
+                # block scalar: consume the indented run beneath it
+                body, i = [], i + 1
+                while i < len(lines) and (not lines[i].strip() or lines[i][:1] in (' ', '\t')):
+                    body.append(lines[i].strip())
+                    i += 1
+                joined = ' '.join(x for x in body if x)
+                if joined:
+                    out[key] = joined
+                continue
+            out[key] = val.strip('"\'')
+            i += 1
+        return out
+
+
 # ── Tools & Commands Scanner ──────────────────────────────────────────────────
 
 class ToolsScanner:
@@ -827,6 +991,22 @@ class ToolsScanner:
                 if (skill_dir / 'SKILL.md').exists():
                     tools.append({'name': f'/{skill_dir.name}', 'command': f'/{skill_dir.name}', 'description': 'Claude skill', 'source': 'skills'})
 
+        # Slash commands — commands/*.md and .claude/commands/*.md
+        seen = {t['name'] for t in tools}
+        for manifest in SkillParser().parse():
+            if manifest['kind'] != 'command':
+                continue
+            name = f"/{manifest['name']}"
+            if name in seen:
+                continue
+            seen.add(name)
+            tools.append({
+                'name': name,
+                'command': name,
+                'description': manifest['description'] or 'slash command',
+                'source': 'commands',
+            })
+
         return tools
 
 
@@ -958,7 +1138,11 @@ def build_vocabulary_section(vocab: list[dict]) -> str:
         notes    = v.get('notes', '').replace('|', '\\|')
         lines.append(f"| {alias} | {type_} | {location} | {notes} |")
     if not vocab:
-        lines.append("| _(no vocabulary generated yet — add source code to populate)_ | | | |")
+        # Prose, not a table row: a placeholder row parses as a valid entry with
+        # an empty (=neutral) location, which scored an empty vocabulary at 100%
+        # and left grader.py's greenfield branch permanently unreachable.
+        lines.append("")
+        lines.append("_No vocabulary generated yet — add source code to populate._")
     return '\n'.join(lines) + '\n'
 
 
@@ -1260,20 +1444,9 @@ def build_dead_code_section(candidates: list[dict]) -> str:
 
 def build_doc_pointers_section() -> str:
     lines = ["# Section 19 — Documentation Pointers\n\n"]
-    docs = []
-    doc_dirs = ['docs', 'doc', 'documentation', 'wiki', '.docs']
-    doc_exts = {'.md', '.rst', '.txt', '.adoc'}
-
-    for doc_dir in doc_dirs:
-        d = PROJECT_ROOT / doc_dir
-        if d.is_dir():
-            for f in sorted(d.rglob('*')):
-                if f.is_file() and f.suffix in doc_exts:
-                    docs.append(str(f.relative_to(PROJECT_ROOT)))
-
-    # Root-level docs
-    for f in PROJECT_ROOT.glob('*.md'):
-        docs.append(str(f.relative_to(PROJECT_ROOT)))
+    # Same walk the checksum watches (DOC_DIRS/DOC_EXTS), so the pointer list and
+    # the watch set cannot drift apart.
+    docs = [str(f.relative_to(PROJECT_ROOT)) for f in collect_doc_files()]
 
     if not docs:
         lines.append("_No documentation files found._\n")
@@ -1283,6 +1456,50 @@ def build_doc_pointers_section() -> str:
     return '\n'.join(lines) + '\n'
 
 
+# ── Glossary side-channel ─────────────────────────────────────────────────────
+
+def _relative_source() -> str:
+    vocab_md = SECTIONS_DIR / '01-vocabulary.md'
+    try:
+        return str(vocab_md.relative_to(PROJECT_ROOT))
+    except ValueError:
+        return str(vocab_md)
+
+
+def write_glossary(vocab: list[dict], stack: dict) -> Path:
+    """Structured vocabulary for machine consumers.
+
+    01-vocabulary.md stays human-facing. Consumers read this instead of parsing
+    prose: the markdown is a rendered table whose Notes column mixes
+    descriptions with metadata and carries pipe-escaping (`a \\| b`), which is
+    fine to read and hostile to parse.
+    """
+    entries = [
+        {
+            'key': v.get('alias', ''),
+            'canonical_path': v.get('location', ''),
+            'section': v.get('type', ''),
+            'description': v.get('notes', '') or None,
+        }
+        for v in sorted(vocab, key=lambda x: x.get('alias', ''))
+        if v.get('alias')
+    ]
+    payload = {
+        'schema_version': GLOSSARY_SCHEMA_VERSION,
+        'generated_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+        # SECTIONS_DIR is bound to the script's location, which is not under
+        # PROJECT_ROOT when --project-root points elsewhere. Report a relative
+        # path when there is one, the absolute path otherwise.
+        'source': _relative_source(),
+        'project': stack.get('name', PROJECT_ROOT.name),
+        'entry_count': len(entries),
+        'entries': entries,
+    }
+    GLOSSARY.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + '\n',
+                        encoding='utf-8')
+    return GLOSSARY
+
+
 # ╔══════════════════════════════════════════════════════════════════════════╗
 # ║  PROJECT MAP TOC                                                         ║
 # ╚══════════════════════════════════════════════════════════════════════════╝
@@ -1297,6 +1514,7 @@ def build_project_map(
     services: list[dict],
     vocab: list[dict],
     section_files: list[tuple[str, Path]],
+    scanned: dict[str, int] | None = None,
 ) -> str:
     now = datetime.now().strftime('%Y-%m-%d %H:%M')
     name = stack.get('name', PROJECT_ROOT.name)
@@ -1315,6 +1533,16 @@ def build_project_map(
         f"| Docker Services | {len(services)} |",
         f"| Vocabulary Entries | {len(vocab)} |",
         f"| Stack | {stack.get('language','?')} / {stack.get('framework','?')} |",
+        "",
+        # Inputs beside outputs. "0 routes" alone cannot be judged: from 47
+        # Python files it means an extractor is broken, from 0 it is correct.
+        # grader.py reads these to tell those two cases apart.
+        "### Source files scanned\n",
+        "| Language | Files |",
+        "|----------|-------|",
+    ] + [
+        f"| {lang} | {count} |" for lang, count in sorted((scanned or {}).items())
+    ] + [
         "",
         "## Section Index\n",
         "| # | Section | Size | When to Read |",
@@ -1392,7 +1620,7 @@ def main() -> None:
 
     # 1. Checksum check
     watched = collect_watched_files()
-    checksum = compute_checksum(watched)
+    checksum = compute_checksum(watched, collect_doc_files())
 
     if not args.force and is_unchanged(checksum):
         print("[generate] ✓ No changes detected — skipping regeneration (use --force to override)")
@@ -1434,12 +1662,16 @@ def main() -> None:
     env_entries = EnvParser().parse()
     migrations = MigrationParser().parse()
     features = FrontendScanner().scan()
+    skills = SkillParser().parse()
     tools = ToolsScanner().scan()
     auth_info = AuthScanner().scan()
     proxy_rules = ReverseProxyScanner().scan()
 
     print("[generate] Building vocabulary...")
-    vocab = VocabularyBuilder().build(routes, models, schemas, features, stack)
+    vocab = VocabularyBuilder().build(routes, models, schemas, features, stack, skills)
+
+    glossary_path = write_glossary(vocab, stack)
+    print(f"[generate] Glossary → {glossary_path.name} ({len(vocab)} entries)")
 
     print("[generate] Tracing import chains...")
     chains = ImportChainTracer().trace(routes)
@@ -1473,7 +1705,11 @@ def main() -> None:
 
     # 6. Write PROJECT_MAP.md
     print("[generate] Writing PROJECT_MAP.md...")
-    project_map = build_project_map(stack, routes, models, schemas, features, migrations, services, vocab, section_files)
+    project_map = build_project_map(
+        stack, routes, models, schemas, features, migrations, services, vocab, section_files,
+        scanned={'python': len(py_files), 'typescript/javascript': len(ts_files),
+                 'go': len(go_files)},
+    )
     (MAP_DIR / 'PROJECT_MAP.md').write_text(project_map, encoding='utf-8')
 
     # 7. Update checksums

package/.claude/project-map/grader.py

@@ -173,7 +173,7 @@ def grade_import_chains() -> GradeResult:
 
     content = chains_file.read_text(encoding='utf-8', errors='replace')
 
-    if '_No import chains traced' in content or '_No' in content:
+    if '_No import chains traced' in content:
         # For greenfield or non-Python: acceptable
         details = "No chains traced (greenfield or non-Python stack — ok)"
         return GradeResult('import_chain_validity', WEIGHTS['import_chain_validity'],
@@ -497,6 +497,17 @@ def print_terminal_summary(results: list[GradeResult], total_score: float, passe
     print(f"  {'TOTAL SCORE':<30} {score_color}{total_score:5.1f}%{RESET}  → {verdict}")
     print(f"{CYAN}{'─' * 55}{RESET}\n")
 
+    pop, total_sections = populated_sections()
+    if total_sections:
+        print(f"  {'Sections populated':<30} {pop}/{total_sections}"
+              f"  (diagnostic — not scored)")
+        print()
+
+    for w in usefulness_warnings():
+        print(f"{YELLOW}  ⚠ {w}{RESET}")
+    if usefulness_warnings():
+        print()
+
     all_issues = [(r.display_name, i) for r in results for i in r.issues]
     if all_issues:
         print(f"{YELLOW}  Issues:{RESET}")
@@ -507,6 +518,71 @@ def print_terminal_summary(results: list[GradeResult], total_score: float, passe
         print()
 
 
+# ── Usefulness diagnostics (reported, never scored) ──────────────────────────
+# The seven graded categories all measure FORM, and generate.py always emits
+# well-formed output — so an empty map and a populated one score identically
+# (both 97.0% before this was added). The information that separates them is
+# not in the map, so it is surfaced as a warning rather than folded into the
+# score: making it scored would fail legitimately sparse repos, which is a
+# worse failure than the one it fixes.
+
+def _stat(content: str, label: str) -> int | None:
+    m = re.search(rf'^\|\s*{re.escape(label)}\s*\|\s*(\d+)\s*\|', content, re.MULTILINE)
+    return int(m.group(1)) if m else None
+
+
+def usefulness_warnings() -> list[str]:
+    map_file = MAP_DIR / 'PROJECT_MAP.md'
+    if not map_file.exists():
+        return []
+    content = map_file.read_text(encoding='utf-8', errors='replace')
+    warnings = []
+
+    # Vocabulary is the product. Zero entries means the map gave the user
+    # nothing, whatever the seven form categories say. This is the signal that
+    # would have surfaced issue #6 at install time.
+    vocab = _stat(content, 'Vocabulary Entries')
+    if vocab == 0:
+        warnings.append(
+            "Vocabulary is empty — the map's primary output produced nothing. "
+            "Expected on a repo with no source code; otherwise an extractor "
+            "does not understand this project's shape."
+        )
+
+    # Source files present but nothing extracted from them: a parser that does
+    # not fit the stack, rather than a repo with nothing to find.
+    scanned = 0
+    block = re.search(r'### Source files scanned(.*?)(?:\n## |\Z)', content, re.DOTALL)
+    if block:
+        scanned = sum(int(n) for n in re.findall(r'^\|[^|]+\|\s*(\d+)\s*\|',
+                                                 block.group(1), re.MULTILINE))
+    extracted = sum(v or 0 for v in (
+        _stat(content, 'API Routes'), _stat(content, 'Data Models'),
+        _stat(content, 'Schemas/DTOs'), _stat(content, 'Frontend Features')))
+    if scanned >= 10 and extracted == 0:
+        warnings.append(
+            f"Scanned {scanned} source file(s) but extracted no routes, models, "
+            f"schemas or features — likely an unsupported framework."
+        )
+    return warnings
+
+
+def populated_sections() -> tuple[int, int]:
+    """Sections carrying real content. Diagnostic only — 19 is aspirational,
+    not a target: a plugin repo can never populate routes or migrations."""
+    files = sorted(SECTIONS_DIR.glob('*.md')) if SECTIONS_DIR.exists() else []
+    pop = 0
+    for f in files:
+        body = f.read_text(encoding='utf-8', errors='replace')
+        meat = [ln for ln in body.splitlines()
+                if ln.strip() and not ln.startswith(('#', '>'))
+                and not re.fullmatch(r'\|[\s|:-]*\|', ln.strip())
+                and not re.match(r'^_\(?(no|none|not)\b', ln.strip(), re.I)]
+        if meat:
+            pop += 1
+    return pop, len(files)
+
+
 # ╔══════════════════════════════════════════════════════════════════════════╗
 # ║  MAIN                                                                    ║
 # ╚══════════════════════════════════════════════════════════════════════════╝