@zeyue0329/xiaoma-cli 1.0.49 → 1.0.66

package/src/core/skills/xiaoma-distillator/scripts/analyze_sources.py

@@ -0,0 +1,300 @@
+# /// script
+# /// requires-python = ">=3.10"
+# /// dependencies = []
+# ///
+"""Analyze source documents for the distillation generator.
+
+Enumerates files from paths/folders/globs, computes sizes and token estimates,
+detects document types from naming conventions, and suggests groupings for
+related documents (e.g., a brief paired with its discovery notes).
+
+Accepts: file paths, folder paths (scans recursively for .md/.txt/.yaml/.yml/.json),
+or glob patterns. Skips node_modules, .git, __pycache__, .venv, _bmad-output.
+
+Output JSON structure:
+  status: "ok" | "error"
+  files[]: path, filename, size_bytes, estimated_tokens, doc_type
+  summary: total_files, total_size_bytes, total_estimated_tokens
+  groups[]: group_key, files[] with role (primary/companion/standalone)
+    - Groups related docs by naming convention (e.g., brief + discovery-notes)
+  routing: recommendation ("single" | "fan-out"), reason
+    - single: ≤3 files AND ≤15K estimated tokens
+    - fan-out: >3 files OR >15K estimated tokens
+  split_prediction: prediction ("likely" | "unlikely"), reason, estimated_distillate_tokens
+    - Estimates distillate at ~1/3 source size; splits if >5K tokens
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+# Extensions to include when scanning folders
+INCLUDE_EXTENSIONS = {".md", ".txt", ".yaml", ".yml", ".json"}
+
+# Directories to skip when scanning folders
+SKIP_DIRS = {
+    "node_modules", ".git", "__pycache__", ".venv", "venv",
+    ".claude", "_bmad-output", ".cursor", ".vscode",
+}
+
+# Approximate chars per token for estimation
+CHARS_PER_TOKEN = 4
+
+# Thresholds
+SINGLE_COMPRESSOR_MAX_TOKENS = 15_000
+SINGLE_DISTILLATE_MAX_TOKENS = 5_000
+
+# Naming patterns for document type detection
+DOC_TYPE_PATTERNS = [
+    (r"discovery[_-]notes", "discovery-notes"),
+    (r"product[_-]brief", "product-brief"),
+    (r"research[_-]report", "research-report"),
+    (r"architecture", "architecture-doc"),
+    (r"prd", "prd"),
+    (r"distillate", "distillate"),
+    (r"changelog", "changelog"),
+    (r"readme", "readme"),
+    (r"spec", "specification"),
+    (r"requirements", "requirements"),
+    (r"design[_-]doc", "design-doc"),
+    (r"meeting[_-]notes", "meeting-notes"),
+    (r"brainstorm", "brainstorming"),
+    (r"interview", "interview-notes"),
+]
+
+# Patterns for grouping related documents
+GROUP_PATTERNS = [
+    # base document + discovery notes
+    (r"^(.+?)(?:-discovery-notes|-discovery_notes)\.(\w+)$", r"\1.\2"),
+    # base document + appendix
+    (r"^(.+?)(?:-appendix|-addendum)(?:-\w+)?\.(\w+)$", r"\1.\2"),
+    # base document + review/feedback
+    (r"^(.+?)(?:-review|-feedback)\.(\w+)$", r"\1.\2"),
+]
+
+
+def resolve_inputs(inputs: list[str]) -> list[Path]:
+    """Resolve input arguments to a flat list of file paths."""
+    files: list[Path] = []
+    for inp in inputs:
+        path = Path(inp)
+        if path.is_file():
+            files.append(path.resolve())
+        elif path.is_dir():
+            for root, dirs, filenames in os.walk(path):
+                dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+                for fn in sorted(filenames):
+                    fp = Path(root) / fn
+                    if fp.suffix.lower() in INCLUDE_EXTENSIONS:
+                        files.append(fp.resolve())
+        else:
+            # Try as glob
+            matches = glob.glob(inp, recursive=True)
+            for m in sorted(matches):
+                mp = Path(m)
+                if mp.is_file() and mp.suffix.lower() in INCLUDE_EXTENSIONS:
+                    files.append(mp.resolve())
+    # Deduplicate while preserving order
+    seen: set[Path] = set()
+    deduped: list[Path] = []
+    for f in files:
+        if f not in seen:
+            seen.add(f)
+            deduped.append(f)
+    return deduped
+
+
+def detect_doc_type(filename: str) -> str:
+    """Detect document type from filename."""
+    name_lower = filename.lower()
+    for pattern, doc_type in DOC_TYPE_PATTERNS:
+        if re.search(pattern, name_lower):
+            return doc_type
+    return "unknown"
+
+
+def suggest_groups(files: list[Path]) -> list[dict]:
+    """Suggest document groupings based on naming conventions."""
+    groups: dict[str, list[dict]] = {}
+    ungrouped: list[dict] = []
+
+    file_map = {f.name: f for f in files}
+
+    assigned: set[str] = set()
+
+    for f in files:
+        if f.name in assigned:
+            continue
+
+        matched = False
+        for pattern, base_pattern in GROUP_PATTERNS:
+            m = re.match(pattern, f.name, re.IGNORECASE)
+            if m:
+                # This file is a companion — find its base
+                base_name = re.sub(pattern, base_pattern, f.name, flags=re.IGNORECASE)
+                group_key = base_name
+                if group_key not in groups:
+                    groups[group_key] = []
+                    # Add the base file if it exists
+                    if base_name in file_map and base_name not in assigned:
+                        groups[group_key].append({
+                            "path": str(file_map[base_name]),
+                            "filename": base_name,
+                            "role": "primary",
+                        })
+                        assigned.add(base_name)
+                groups[group_key].append({
+                    "path": str(f),
+                    "filename": f.name,
+                    "role": "companion",
+                })
+                assigned.add(f.name)
+                matched = True
+                break
+
+        if not matched:
+            # Check if this file is a base that already has companions
+            if f.name in groups:
+                continue  # Already added as primary
+            ungrouped.append({
+                "path": str(f),
+                "filename": f.name,
+            })
+
+    result = []
+    for group_key, members in groups.items():
+        result.append({
+            "group_key": group_key,
+            "files": members,
+        })
+    for ug in ungrouped:
+        if ug["filename"] not in assigned:
+            result.append({
+                "group_key": ug["filename"],
+                "files": [{"path": ug["path"], "filename": ug["filename"], "role": "standalone"}],
+            })
+
+    return result
+
+
+def analyze(inputs: list[str], output_path: str | None = None) -> None:
+    """Main analysis function."""
+    files = resolve_inputs(inputs)
+
+    if not files:
+        result = {
+            "status": "error",
+            "error": "No readable files found from provided inputs",
+            "inputs": inputs,
+        }
+        output_json(result, output_path)
+        return
+
+    # Analyze each file
+    file_details = []
+    total_chars = 0
+    for f in files:
+        size = f.stat().st_size
+        total_chars += size
+        file_details.append({
+            "path": str(f),
+            "filename": f.name,
+            "size_bytes": size,
+            "estimated_tokens": size // CHARS_PER_TOKEN,
+            "doc_type": detect_doc_type(f.name),
+        })
+
+    total_tokens = total_chars // CHARS_PER_TOKEN
+    groups = suggest_groups(files)
+
+    # Routing recommendation
+    if len(files) <= 3 and total_tokens <= SINGLE_COMPRESSOR_MAX_TOKENS:
+        routing = "single"
+        routing_reason = (
+            f"{len(files)} file(s), ~{total_tokens:,} estimated tokens — "
+            f"within single compressor threshold"
+        )
+    else:
+        routing = "fan-out"
+        routing_reason = (
+            f"{len(files)} file(s), ~{total_tokens:,} estimated tokens — "
+            f"exceeds single compressor threshold "
+            f"({'>' + str(SINGLE_COMPRESSOR_MAX_TOKENS) + ' tokens' if total_tokens > SINGLE_COMPRESSOR_MAX_TOKENS else '> 3 files'})"
+        )
+
+    # Split prediction
+    estimated_distillate_tokens = total_tokens // 3  # rough: distillate is ~1/3 of source
+    if estimated_distillate_tokens > SINGLE_DISTILLATE_MAX_TOKENS:
+        split_prediction = "likely"
+        split_reason = (
+            f"Estimated distillate ~{estimated_distillate_tokens:,} tokens "
+            f"exceeds {SINGLE_DISTILLATE_MAX_TOKENS:,} threshold"
+        )
+    else:
+        split_prediction = "unlikely"
+        split_reason = (
+            f"Estimated distillate ~{estimated_distillate_tokens:,} tokens "
+            f"within {SINGLE_DISTILLATE_MAX_TOKENS:,} threshold"
+        )
+
+    result = {
+        "status": "ok",
+        "files": file_details,
+        "summary": {
+            "total_files": len(files),
+            "total_size_bytes": total_chars,
+            "total_estimated_tokens": total_tokens,
+        },
+        "groups": groups,
+        "routing": {
+            "recommendation": routing,
+            "reason": routing_reason,
+        },
+        "split_prediction": {
+            "prediction": split_prediction,
+            "reason": split_reason,
+            "estimated_distillate_tokens": estimated_distillate_tokens,
+        },
+    }
+
+    output_json(result, output_path)
+
+
+def output_json(data: dict, output_path: str | None) -> None:
+    """Write JSON to file or stdout."""
+    json_str = json.dumps(data, indent=2)
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        Path(output_path).write_text(json_str + "\n")
+        print(f"Results written to {output_path}", file=sys.stderr)
+    else:
+        print(json_str)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "inputs",
+        nargs="+",
+        help="File paths, folder paths, or glob patterns to analyze",
+    )
+    parser.add_argument(
+        "-o", "--output",
+        help="Output JSON to file instead of stdout",
+    )
+    args = parser.parse_args()
+    analyze(args.inputs, args.output)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

package/src/core/skills/xiaoma-distillator/scripts/tests/test_analyze_sources.py

@@ -0,0 +1,204 @@
+"""Tests for analyze_sources.py"""
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+# Add parent dir to path so we can import the script
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from analyze_sources import (
+    resolve_inputs,
+    detect_doc_type,
+    suggest_groups,
+    analyze,
+    INCLUDE_EXTENSIONS,
+    SKIP_DIRS,
+)
+
+
+@pytest.fixture
+def temp_dir():
+    """Create a temp directory with sample files."""
+    with tempfile.TemporaryDirectory() as d:
+        # Create sample files
+        (Path(d) / "product-brief-foo.md").write_text("# Product Brief\nContent here")
+        (Path(d) / "product-brief-foo-discovery-notes.md").write_text("# Discovery\nNotes")
+        (Path(d) / "architecture-doc.md").write_text("# Architecture\nDesign here")
+        (Path(d) / "research-report.md").write_text("# Research\nFindings")
+        (Path(d) / "random.txt").write_text("Some text content")
+        (Path(d) / "image.png").write_bytes(b"\x89PNG")
+        # Create a subdirectory with more files
+        sub = Path(d) / "subdir"
+        sub.mkdir()
+        (sub / "prd-v2.md").write_text("# PRD\nRequirements")
+        # Create a skip directory
+        skip = Path(d) / "node_modules"
+        skip.mkdir()
+        (skip / "junk.md").write_text("Should be skipped")
+        yield d
+
+
+class TestResolveInputs:
+    def test_single_file(self, temp_dir):
+        f = str(Path(temp_dir) / "product-brief-foo.md")
+        result = resolve_inputs([f])
+        assert len(result) == 1
+        assert result[0].name == "product-brief-foo.md"
+
+    def test_folder_recursion(self, temp_dir):
+        result = resolve_inputs([temp_dir])
+        names = {f.name for f in result}
+        assert "product-brief-foo.md" in names
+        assert "prd-v2.md" in names
+        assert "random.txt" in names
+
+    def test_folder_skips_excluded_dirs(self, temp_dir):
+        result = resolve_inputs([temp_dir])
+        names = {f.name for f in result}
+        assert "junk.md" not in names
+
+    def test_folder_skips_non_text_files(self, temp_dir):
+        result = resolve_inputs([temp_dir])
+        names = {f.name for f in result}
+        assert "image.png" not in names
+
+    def test_glob_pattern(self, temp_dir):
+        pattern = str(Path(temp_dir) / "product-brief-*.md")
+        result = resolve_inputs([pattern])
+        assert len(result) == 2
+        names = {f.name for f in result}
+        assert "product-brief-foo.md" in names
+        assert "product-brief-foo-discovery-notes.md" in names
+
+    def test_deduplication(self, temp_dir):
+        f = str(Path(temp_dir) / "product-brief-foo.md")
+        result = resolve_inputs([f, f, f])
+        assert len(result) == 1
+
+    def test_mixed_inputs(self, temp_dir):
+        file_path = str(Path(temp_dir) / "architecture-doc.md")
+        folder_path = str(Path(temp_dir) / "subdir")
+        result = resolve_inputs([file_path, folder_path])
+        names = {f.name for f in result}
+        assert "architecture-doc.md" in names
+        assert "prd-v2.md" in names
+
+    def test_nonexistent_path(self):
+        result = resolve_inputs(["/nonexistent/path/file.md"])
+        assert len(result) == 0
+
+
+class TestDetectDocType:
+    @pytest.mark.parametrize("filename,expected", [
+        ("product-brief-foo.md", "product-brief"),
+        ("product_brief_bar.md", "product-brief"),
+        ("foo-discovery-notes.md", "discovery-notes"),
+        ("foo-discovery_notes.md", "discovery-notes"),
+        ("architecture-overview.md", "architecture-doc"),
+        ("my-prd.md", "prd"),
+        ("research-report-q4.md", "research-report"),
+        ("foo-distillate.md", "distillate"),
+        ("changelog.md", "changelog"),
+        ("readme.md", "readme"),
+        ("api-spec.md", "specification"),
+        ("design-doc-v2.md", "design-doc"),
+        ("meeting-notes-2026.md", "meeting-notes"),
+        ("brainstorm-session.md", "brainstorming"),
+        ("user-interview-notes.md", "interview-notes"),
+        ("random-file.md", "unknown"),
+    ])
+    def test_detection(self, filename, expected):
+        assert detect_doc_type(filename) == expected
+
+
+class TestSuggestGroups:
+    def test_groups_brief_with_discovery_notes(self, temp_dir):
+        files = [
+            Path(temp_dir) / "product-brief-foo.md",
+            Path(temp_dir) / "product-brief-foo-discovery-notes.md",
+        ]
+        groups = suggest_groups(files)
+        # Should produce one group with both files
+        paired = [g for g in groups if len(g["files"]) > 1]
+        assert len(paired) == 1
+        filenames = {f["filename"] for f in paired[0]["files"]}
+        assert "product-brief-foo.md" in filenames
+        assert "product-brief-foo-discovery-notes.md" in filenames
+
+    def test_standalone_files(self, temp_dir):
+        files = [
+            Path(temp_dir) / "architecture-doc.md",
+            Path(temp_dir) / "research-report.md",
+        ]
+        groups = suggest_groups(files)
+        assert len(groups) == 2
+        for g in groups:
+            assert len(g["files"]) == 1
+
+    def test_mixed_grouped_and_standalone(self, temp_dir):
+        files = [
+            Path(temp_dir) / "product-brief-foo.md",
+            Path(temp_dir) / "product-brief-foo-discovery-notes.md",
+            Path(temp_dir) / "architecture-doc.md",
+        ]
+        groups = suggest_groups(files)
+        paired = [g for g in groups if len(g["files"]) > 1]
+        standalone = [g for g in groups if len(g["files"]) == 1]
+        assert len(paired) == 1
+        assert len(standalone) == 1
+
+
+class TestAnalyze:
+    def test_basic_analysis(self, temp_dir):
+        f = str(Path(temp_dir) / "product-brief-foo.md")
+        output_file = str(Path(temp_dir) / "output.json")
+        analyze([f], output_file)
+        result = json.loads(Path(output_file).read_text())
+        assert result["status"] == "ok"
+        assert result["summary"]["total_files"] == 1
+        assert result["files"][0]["doc_type"] == "product-brief"
+        assert result["files"][0]["estimated_tokens"] > 0
+
+    def test_routing_single_small_input(self, temp_dir):
+        f = str(Path(temp_dir) / "product-brief-foo.md")
+        output_file = str(Path(temp_dir) / "output.json")
+        analyze([f], output_file)
+        result = json.loads(Path(output_file).read_text())
+        assert result["routing"]["recommendation"] == "single"
+
+    def test_routing_fanout_many_files(self, temp_dir):
+        # Create enough files to trigger fan-out (> 3 files)
+        for i in range(5):
+            (Path(temp_dir) / f"doc-{i}.md").write_text("x" * 1000)
+        output_file = str(Path(temp_dir) / "output.json")
+        analyze([temp_dir], output_file)
+        result = json.loads(Path(output_file).read_text())
+        assert result["routing"]["recommendation"] == "fan-out"
+
+    def test_folder_analysis(self, temp_dir):
+        output_file = str(Path(temp_dir) / "output.json")
+        analyze([temp_dir], output_file)
+        result = json.loads(Path(output_file).read_text())
+        assert result["status"] == "ok"
+        assert result["summary"]["total_files"] >= 4  # at least the base files
+        assert len(result["groups"]) > 0
+
+    def test_no_files_found(self):
+        output_file = "/tmp/test_analyze_empty.json"
+        analyze(["/nonexistent/path"], output_file)
+        result = json.loads(Path(output_file).read_text())
+        assert result["status"] == "error"
+        os.unlink(output_file)
+
+    def test_stdout_output(self, temp_dir, capsys):
+        f = str(Path(temp_dir) / "product-brief-foo.md")
+        analyze([f])
+        captured = capsys.readouterr()
+        result = json.loads(captured.out)
+        assert result["status"] == "ok"

package/src/core/skills/xiaoma-distillator/xiaoma-skill-manifest.yaml

@@ -0,0 +1,15 @@
+type: skill
+module: core
+capabilities:
+  - name: xiaoma-distillator
+    menu-code: DSTL
+    description: "Produces lossless LLM-optimized distillate from source documents. Use after producing large human presentable documents that will be consumed later by LLMs"
+    supports-headless: true
+    input: source documents
+    args: output, validate
+    output: single distillate or folder of distillates next to source input
+    config-vars-used: null
+    phase: anytime
+    before: []
+    after: []
+    is-required: false

package/src/core/skills/xiaoma-editorial-review-prose/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: xiaoma-editorial-review-prose
+description: 'Clinical copy-editor that reviews text for communication issues. Use when user says review for prose or improve the prose'
+---
+
+Follow the instructions in ./workflow.md.

package/src/core/skills/xiaoma-editorial-review-prose/workflow.md

@@ -0,0 +1,81 @@
+# Editorial Review - Prose
+
+**Goal:** Review text for communication issues that impede comprehension and output suggested fixes in a three-column table.
+
+**Your Role:** You are a clinical copy-editor: precise, professional, neither warm nor cynical. Apply Microsoft Writing Style Guide principles as your baseline. Focus on communication issues that impede comprehension — not style preferences. NEVER rewrite for preference — only fix genuine issues. Follow ALL steps in the STEPS section IN EXACT ORDER. DO NOT skip steps or change the sequence. HALT immediately when halt-conditions are met. Each action within a step is a REQUIRED action to complete that step.
+
+**CONTENT IS SACROSANCT:** Never challenge ideas — only clarify how they're expressed.
+
+**Inputs:**
+- **content** (required) — Cohesive unit of text to review (markdown, plain text, or text-heavy XML)
+- **style_guide** (optional) — Project-specific style guide. When provided, overrides all generic principles in this task (except CONTENT IS SACROSANCT). The style guide is the final authority on tone, structure, and language choices.
+- **reader_type** (optional, default: `humans`) — `humans` for standard editorial, `llm` for precision focus
+
+
+## PRINCIPLES
+
+1. **Minimal intervention:** Apply the smallest fix that achieves clarity
+2. **Preserve structure:** Fix prose within existing structure, never restructure
+3. **Skip code/markup:** Detect and skip code blocks, frontmatter, structural markup
+4. **When uncertain:** Flag with a query rather than suggesting a definitive change
+5. **Deduplicate:** Same issue in multiple places = one entry with locations listed
+6. **No conflicts:** Merge overlapping fixes into single entries
+7. **Respect author voice:** Preserve intentional stylistic choices
+
+> **STYLE GUIDE OVERRIDE:** If a style_guide input is provided, it overrides ALL generic principles in this task (including the Microsoft Writing Style Guide baseline and reader_type-specific priorities). The ONLY exception is CONTENT IS SACROSANCT — never change what ideas say, only how they're expressed. When style guide conflicts with this task, style guide wins.
+
+
+## STEPS
+
+### Step 1: Validate Input
+
+- Check if content is empty or contains fewer than 3 words
+  - If empty or fewer than 3 words: **HALT** with error: "Content too short for editorial review (minimum 3 words required)"
+- Validate reader_type is `humans` or `llm` (or not provided, defaulting to `humans`)
+  - If reader_type is invalid: **HALT** with error: "Invalid reader_type. Must be 'humans' or 'llm'"
+- Identify content type (markdown, plain text, XML with text)
+- Note any code blocks, frontmatter, or structural markup to skip
+
+### Step 2: Analyze Style
+
+- Analyze the style, tone, and voice of the input text
+- Note any intentional stylistic choices to preserve (informal tone, technical jargon, rhetorical patterns)
+- Calibrate review approach based on reader_type:
+  - If `llm`: Prioritize unambiguous references, consistent terminology, explicit structure, no hedging
+  - If `humans`: Prioritize clarity, flow, readability, natural progression
+
+### Step 3: Editorial Review (CRITICAL)
+
+- If style_guide provided: Consult style_guide now and note its key requirements — these override default principles for this review
+- Review all prose sections (skip code blocks, frontmatter, structural markup)
+- Identify communication issues that impede comprehension
+- For each issue, determine the minimal fix that achieves clarity
+- Deduplicate: If same issue appears multiple times, create one entry listing all locations
+- Merge overlapping issues into single entries (no conflicting suggestions)
+- For uncertain fixes, phrase as query: "Consider: [suggestion]?" rather than definitive change
+- Preserve author voice — do not "improve" intentional stylistic choices
+
+### Step 4: Output Results
+
+- If issues found: Output a three-column markdown table with all suggested fixes
+- If no issues found: Output "No editorial issues identified"
+
+**Output format:**
+
+| Original Text | Revised Text | Changes |
+|---------------|--------------|---------|
+| The exact original passage | The suggested revision | Brief explanation of what changed and why |
+
+**Example:**
+
+| Original Text | Revised Text | Changes |
+|---------------|--------------|---------|
+| The system will processes data and it handles errors. | The system processes data and handles errors. | Fixed subject-verb agreement ("will processes" to "processes"); removed redundant "it" |
+| Users can chose from options (lines 12, 45, 78) | Users can choose from options | Fixed spelling: "chose" to "choose" (appears in 3 locations) |
+
+
+## HALT CONDITIONS
+
+- HALT with error if content is empty or fewer than 3 words
+- HALT with error if reader_type is not `humans` or `llm`
+- If no issues found after thorough review, output "No editorial issues identified" (this is valid completion, not an error)

package/src/core/skills/xiaoma-editorial-review-prose/xiaoma-skill-manifest.yaml

@@ -0,0 +1 @@
+type: skill

package/src/core/skills/xiaoma-editorial-review-structure/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: xiaoma-editorial-review-structure
+description: 'Structural editor that proposes cuts, reorganization, and simplification while preserving comprehension. Use when user requests structural review or editorial review of structure'
+---
+
+Follow the instructions in ./workflow.md.