learn_bash_from_session_data 1.0.9 → 1.1.0

package/scripts/main.py

@@ -22,6 +22,7 @@ if sys.version_info < (3, 8):
 # Constants
 DEFAULT_OUTPUT_BASE = "./bash-learner-output"
 MAX_UNIQUE_COMMANDS = 500
+VERSION = "1.0.10"
 
 
 def generate_timestamped_output_dir(base_dir: str = DEFAULT_OUTPUT_BASE) -> Path:
@@ -51,7 +52,8 @@ def get_sessions_base_path() -> Path:
     is_wsl = False
     try:
         with open("/proc/version", "r") as f:
-            is_wsl = "microsoft" in f.read().lower() or "wsl" in f.read().lower()
+            proc_version = f.read().lower()
+            is_wsl = "microsoft" in proc_version or "wsl" in proc_version
     except (FileNotFoundError, PermissionError):
         pass
 
@@ -374,7 +376,7 @@ def run_extraction_pipeline(
     # Step 5: Re-parse expanded commands to get proper base_command for each
     parsed_expanded = parse_commands(expanded_commands)
 
-    # Step 6: Count frequencies BEFORE deduplication
+    # Step 6: Count frequencies BEFORE deduplication (for accurate usage stats)
     cmd_frequency = Counter()
     base_cmd_frequency = Counter()
 
@@ -386,7 +388,7 @@ def run_extraction_pipeline(
         if base_cmd:
             base_cmd_frequency[base_cmd] += 1
 
-    # Step 7: Deduplicate and add frequency data
+    # Step 7: Deduplicate and attach frequency data
     unique_commands = deduplicate_commands(parsed_expanded)
 
     # Add frequency to each unique command
@@ -403,7 +405,7 @@ def run_extraction_pipeline(
     else:
         print(f"\n{len(unique_commands)} unique commands")
 
-    # Step 6: Analyze commands
+    # Step 8: Analyze commands
     print("\nAnalyzing commands...")
     analysis = analyze_commands(unique_commands)
 
@@ -415,13 +417,13 @@ def run_extraction_pipeline(
     analysis['operators_used'] = dict(operator_frequency)
     print(f"  -> Generated analysis with {len(analysis.get('categories', {}))} categories")
 
-    # Step 6: Generate quizzes
+    # Step 9: Generate quizzes
     print("\nGenerating quizzes...")
     quizzes = generate_quizzes(unique_commands, analysis)
     quiz_count = sum(len(q) for q in quizzes.values()) if isinstance(quizzes, dict) else len(quizzes)
     print(f"  -> Generated {quiz_count} quiz questions")
 
-    # Step 7: Generate HTML
+    # Step 10: Generate HTML
     print("\nGenerating HTML output...")
     html_files = generate_html(unique_commands, analysis, quizzes, output_dir)
     print(f"  -> Created {len(html_files)} HTML files")
@@ -431,7 +433,7 @@ def run_extraction_pipeline(
         "metadata": {
             "generated_at": datetime.now().isoformat(),
             "run_id": output_dir.name,
-            "version": "1.0.5",
+            "version": VERSION,
         },
         "input": {
             "sessions_processed": len(sessions),
@@ -477,8 +479,8 @@ def extract_sub_commands(cmd_str: str) -> List[str]:
     """
     Extract individual sub-commands from a compound command.
 
-    Splits commands by ||, &&, |, and ; while preserving each sub-command
-    as a learnable unit.
+    Splits commands by ||, &&, |, and ; while respecting quoting
+    and skipping inline code commands (python -c, node -e, bash -c).
 
     Args:
         cmd_str: The compound command string
@@ -488,35 +490,77 @@ def extract_sub_commands(cmd_str: str) -> List[str]:
     """
     import re
 
-    # First, clean up redirections but keep them with their command
-    # We want "pip show pkg 2>/dev/null" to stay together
+    if not cmd_str or not cmd_str.strip():
+        return []
 
-    # Split by compound operators: ||, &&, |, ;
-    # Use regex to split while handling edge cases
-    # Note: | needs special handling to not match ||
+    # Don't split commands that contain inline code - the ; and | inside
+    # quoted code would produce garbage fragments
+    inline_patterns = [' -c "', " -c '", ' -c $', ' -e "', " -e '", ' -e $',
+                       ' -c\n', ' -c\r']
+    first_token = cmd_str.split()[0] if cmd_str.split() else ''
+    if first_token in ('python', 'python3', 'node', 'bash', 'sh', 'ruby', 'perl'):
+        for pat in inline_patterns:
+            if pat in cmd_str:
+                return [cmd_str.strip()]
 
+    # Quote-aware splitting: track quote depth to avoid splitting inside quotes
     sub_commands = []
+    current = []
+    in_single = False
+    in_double = False
+    i = 0
+    chars = cmd_str
+
+    while i < len(chars):
+        c = chars[i]
+
+        # Track quoting state
+        if c == "'" and not in_double:
+            in_single = not in_single
+            current.append(c)
+            i += 1
+        elif c == '"' and not in_single:
+            in_double = not in_double
+            current.append(c)
+            i += 1
+        elif not in_single and not in_double:
+            # Check for compound operators outside quotes
+            remaining = chars[i:]
+            if remaining.startswith('&&'):
+                cmd = ''.join(current).strip()
+                if cmd:
+                    sub_commands.append(cmd)
+                current = []
+                i += 2
+            elif remaining.startswith('||'):
+                cmd = ''.join(current).strip()
+                if cmd:
+                    sub_commands.append(cmd)
+                current = []
+                i += 2
+            elif c == ';':
+                cmd = ''.join(current).strip()
+                if cmd:
+                    sub_commands.append(cmd)
+                current = []
+                i += 1
+            elif c == '|' and not remaining.startswith('||'):
+                cmd = ''.join(current).strip()
+                if cmd:
+                    sub_commands.append(cmd)
+                current = []
+                i += 1
+            else:
+                current.append(c)
+                i += 1
+        else:
+            current.append(c)
+            i += 1
 
-    # Split by || first (highest precedence for our purposes)
-    or_parts = re.split(r'\s*\|\|\s*', cmd_str)
-
-    for or_part in or_parts:
-        # Split each part by &&
-        and_parts = re.split(r'\s*&&\s*', or_part)
-
-        for and_part in and_parts:
-            # Split each part by ; (sequential)
-            seq_parts = re.split(r'\s*;\s*', and_part)
-
-            for seq_part in seq_parts:
-                # Split by single pipe |
-                # Use negative lookbehind/lookahead to avoid ||
-                pipe_parts = re.split(r'(?<!\|)\|(?!\|)', seq_part)
-
-                for pipe_part in pipe_parts:
-                    cleaned = pipe_part.strip()
-                    if cleaned:
-                        sub_commands.append(cleaned)
+    # Add final segment
+    cmd = ''.join(current).strip()
+    if cmd:
+        sub_commands.append(cmd)
 
     return sub_commands
 

package/scripts/merge_enrichment.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Merge enrichment data into knowledge_base.py COMMAND_DB.
+
+Reads enrichment data from enrichment_*.py files and merges them into
+the existing COMMAND_DB entries in knowledge_base.py. Adds missing fields
+(use_cases, gotchas, man_url, related, difficulty) and supplements
+existing flag definitions with extra_flags.
+
+Usage:
+    python scripts/merge_enrichment.py [--dry-run]
+"""
+
+import sys
+import re
+import importlib
+import importlib.util
+from pathlib import Path
+from typing import Dict, Any
+
+
+def load_enrichment_module(filepath: Path) -> Dict[str, Any]:
+    """Load ENRICHMENT_DATA from a Python file."""
+    spec = importlib.util.spec_from_file_location("enrichment", filepath)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return getattr(module, 'ENRICHMENT_DATA', {})
+
+
+def collect_all_enrichments(scripts_dir: Path) -> Dict[str, Any]:
+    """Collect enrichment data from all enrichment_*.py files."""
+    merged = {}
+    for enrichment_file in sorted(scripts_dir.glob("enrichment_*.py")):
+        print(f"  Loading: {enrichment_file.name}")
+        data = load_enrichment_module(enrichment_file)
+        print(f"    -> {len(data)} commands")
+        for cmd_name, cmd_data in data.items():
+            if cmd_name in merged:
+                # Merge: later files can supplement but not overwrite
+                for key, value in cmd_data.items():
+                    if key not in merged[cmd_name] or not merged[cmd_name][key]:
+                        merged[cmd_name][key] = value
+            else:
+                merged[cmd_name] = cmd_data
+    return merged
+
+
+def merge_into_knowledge_base(kb_path: Path, enrichments: Dict[str, Any], dry_run: bool = False) -> int:
+    """
+    Merge enrichment data into knowledge_base.py by modifying COMMAND_DB entries.
+
+    Strategy: For each command in enrichments, find its entry in COMMAND_DB and
+    insert the enrichment fields before the closing brace of that entry.
+
+    Returns number of commands enriched.
+    """
+    content = kb_path.read_text(encoding='utf-8')
+    original_content = content
+    enriched_count = 0
+    fields_to_add = ['man_url', 'use_cases', 'gotchas', 'related', 'difficulty']
+
+    for cmd_name, enrichment in enrichments.items():
+        # Find this command's entry in COMMAND_DB
+        # Pattern: "cmd_name": { ... },
+        # We look for the closing "}, " or "},\n" of this entry
+
+        # Find the start of this command's dict entry
+        # Handle both regular command names and special ones like "."
+        escaped_name = re.escape(cmd_name)
+        entry_pattern = rf'    "{escaped_name}": \{{'
+        match = re.search(entry_pattern, content)
+        if not match:
+            print(f"  WARNING: Command '{cmd_name}' not found in COMMAND_DB, skipping")
+            continue
+
+        entry_start = match.start()
+
+        # Find the closing of this entry by counting braces
+        brace_depth = 0
+        entry_end = -1
+        i = match.end() - 1  # Start at the opening brace
+        while i < len(content):
+            char = content[i]
+            if char == '{':
+                brace_depth += 1
+            elif char == '}':
+                brace_depth -= 1
+                if brace_depth == 0:
+                    entry_end = i
+                    break
+            # Skip string contents to avoid counting braces in strings
+            elif char == '"':
+                i += 1
+                while i < len(content) and content[i] != '"':
+                    if content[i] == '\\':
+                        i += 1  # Skip escaped char
+                    i += 1
+            elif char == "'":
+                i += 1
+                while i < len(content) and content[i] != "'":
+                    if content[i] == '\\':
+                        i += 1
+                    i += 1
+            i += 1
+
+        if entry_end == -1:
+            print(f"  WARNING: Could not find end of entry for '{cmd_name}', skipping")
+            continue
+
+        # Extract the entry content
+        entry_content = content[entry_start:entry_end + 1]
+
+        # Check which fields are missing
+        additions = []
+        for field in fields_to_add:
+            if f'"{field}"' not in entry_content:
+                value = enrichment.get(field)
+                if value:
+                    additions.append((field, value))
+
+        # Handle extra_flags: merge into existing flags dict
+        extra_flags = enrichment.get('extra_flags', {})
+        if extra_flags and '"flags"' in entry_content:
+            # Find the flags dict closing brace and add new flags before it
+            flags_additions = []
+            for flag, desc in extra_flags.items():
+                escaped_flag = flag.replace('"', '\\"')
+                if f'"{escaped_flag}"' not in entry_content:
+                    flags_additions.append(f'            "{escaped_flag}": "{desc}",')
+            if flags_additions:
+                # Find the closing of the flags dict within this entry
+                flags_match = re.search(r'"flags":\s*\{', entry_content)
+                if flags_match:
+                    flags_start = flags_match.end()
+                    # Find closing brace of flags
+                    fb_depth = 1
+                    fi = flags_start
+                    while fi < len(entry_content) and fb_depth > 0:
+                        if entry_content[fi] == '{':
+                            fb_depth += 1
+                        elif entry_content[fi] == '}':
+                            fb_depth -= 1
+                        elif entry_content[fi] == '"':
+                            fi += 1
+                            while fi < len(entry_content) and entry_content[fi] != '"':
+                                if entry_content[fi] == '\\':
+                                    fi += 1
+                                fi += 1
+                        fi += 1
+                    flags_end_pos = entry_start + fi - 1
+                    # Insert new flags before the closing brace
+                    flags_insert = '\n' + '\n'.join(flags_additions) + '\n        '
+                    content = content[:flags_end_pos] + flags_insert + content[flags_end_pos:]
+                    # Recalculate entry_end since we modified content
+                    entry_end += len(flags_insert)
+
+        # Handle improved_description: replace existing description
+        improved_desc = enrichment.get('improved_description')
+        if improved_desc and '"description"' in entry_content:
+            # Replace the existing description string
+            desc_pattern = rf'(    "{escaped_name}": \{{[^}}]*?"description":\s*)"([^"]*(?:\\.[^"]*)*)"'
+            new_desc = improved_desc.replace('"', '\\"')
+            content = re.sub(desc_pattern, rf'\1"{new_desc}"', content, count=1)
+
+        if not additions:
+            continue
+
+        # Build the insertion text
+        insertion_lines = []
+        for field, value in additions:
+            if isinstance(value, str):
+                escaped_val = value.replace('"', '\\"')
+                insertion_lines.append(f'        "{field}": "{escaped_val}",')
+            elif isinstance(value, list):
+                if all(isinstance(v, str) for v in value):
+                    items = ', '.join(f'"{v}"' for v in value)
+                    if len(items) < 80:
+                        insertion_lines.append(f'        "{field}": [{items}],')
+                    else:
+                        insertion_lines.append(f'        "{field}": [')
+                        for v in value:
+                            escaped_v = v.replace('"', '\\"')
+                            insertion_lines.append(f'            "{escaped_v}",')
+                        insertion_lines.append(f'        ],')
+
+        if insertion_lines:
+            insertion = '\n' + '\n'.join(insertion_lines)
+            # Recalculate entry_end in current content
+            match2 = re.search(entry_pattern, content)
+            if match2:
+                brace_depth = 0
+                i2 = match2.end() - 1
+                while i2 < len(content):
+                    char = content[i2]
+                    if char == '{':
+                        brace_depth += 1
+                    elif char == '}':
+                        brace_depth -= 1
+                        if brace_depth == 0:
+                            entry_end = i2
+                            break
+                    elif char == '"':
+                        i2 += 1
+                        while i2 < len(content) and content[i2] != '"':
+                            if content[i2] == '\\':
+                                i2 += 1
+                            i2 += 1
+                    elif char == "'":
+                        i2 += 1
+                        while i2 < len(content) and content[i2] != "'":
+                            if content[i2] == '\\':
+                                i2 += 1
+                            i2 += 1
+                    i2 += 1
+
+                # Insert before the closing brace
+                content = content[:entry_end] + insertion + '\n    ' + content[entry_end:]
+                enriched_count += 1
+
+    if content != original_content:
+        if dry_run:
+            print(f"\n  DRY RUN: Would enrich {enriched_count} commands")
+            # Show a diff summary
+            added_lines = len(content.splitlines()) - len(original_content.splitlines())
+            print(f"  Would add ~{added_lines} lines")
+        else:
+            kb_path.write_text(content, encoding='utf-8')
+            print(f"\n  Enriched {enriched_count} commands in {kb_path.name}")
+
+    return enriched_count
+
+
+def main():
+    dry_run = '--dry-run' in sys.argv
+
+    scripts_dir = Path(__file__).parent
+    kb_path = scripts_dir / 'knowledge_base.py'
+
+    if not kb_path.exists():
+        print(f"Error: {kb_path} not found")
+        return 1
+
+    print("Collecting enrichment data...")
+    enrichments = collect_all_enrichments(scripts_dir)
+
+    if not enrichments:
+        print("No enrichment data found. Run the research agents first.")
+        return 1
+
+    print(f"\nTotal enrichments: {len(enrichments)} commands")
+    print(f"\nMerging into {kb_path.name}{'  (DRY RUN)' if dry_run else ''}...")
+    count = merge_into_knowledge_base(kb_path, enrichments, dry_run=dry_run)
+
+    if count > 0:
+        # Verify the file is still valid Python
+        if not dry_run:
+            print("\nVerifying syntax...")
+            try:
+                compile(kb_path.read_text(encoding='utf-8'), kb_path, 'exec')
+                print("  Syntax OK")
+            except SyntaxError as e:
+                print(f"  SYNTAX ERROR: {e}")
+                print("  Reverting changes...")
+                # We'd need to keep a backup for this - for now just warn
+                return 1
+
+    print("\nDone.")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/quiz_generator.py

@@ -784,29 +784,25 @@ def generate_what_does_quiz(
         QuizQuestion instance
     """
     cmd_string = command.get("command", "")
-    description = command.get("description", "")
     complexity = command.get("complexity", 2)
 
     parsed = _parse_command(cmd_string)
     base_cmd = parsed["base"]
 
-    # Build the correct description using educational bash-focused generator
-    correct_desc = description
-    if not correct_desc:
-        # Use the educational bash description generator
-        correct_desc = _generate_bash_description(cmd_string)
-        # Capitalize first letter for consistent formatting
-        if correct_desc:
-            correct_desc = correct_desc[0].upper() + correct_desc[1:]
-
-        # Add flag details if available
-        flag_descs = []
-        for flag in parsed["flags"]:
-            fd = _get_flag_description(base_cmd, flag)
-            if fd:
-                flag_descs.append(f"{flag} ({fd.lower()})")
-        if flag_descs:
-            correct_desc += " using " + ", ".join(flag_descs)
+    # Always use the educational bash description generator (not session descriptions)
+    correct_desc = _generate_bash_description(cmd_string)
+    # Capitalize first letter for consistent formatting
+    if correct_desc:
+        correct_desc = correct_desc[0].upper() + correct_desc[1:]
+
+    # Add flag details if available
+    flag_descs = []
+    for flag in parsed["flags"]:
+        fd = _get_flag_description(base_cmd, flag)
+        if fd:
+            flag_descs.append(f"{flag} ({fd.lower()})")
+    if flag_descs:
+        correct_desc += " using " + ", ".join(flag_descs)
 
     # Generate distractors
     distractor_descriptions = _generate_distractor_descriptions(correct_desc, 3)
@@ -965,8 +961,6 @@ def generate_build_command_quiz(
         QuizQuestion instance
     """
     cmd_string = command.get("command", "")
-    description = command.get("description", "")
-    intent = command.get("intent", description)
 
     parsed = _parse_command(cmd_string)
     base_cmd = parsed["base"]
@@ -1051,14 +1045,8 @@ def generate_build_command_quiz(
 
     question_id = _generate_id(f"build_{cmd_string}")
 
-    # Use educational bash description for task if no intent/description available
-    if intent:
-        task_description = intent
-    elif description:
-        task_description = description
-    else:
-        # Generate educational description from the command
-        task_description = _generate_bash_description(cmd_string)
+    # Always generate description from the command itself (not session descriptions)
+    task_description = _generate_bash_description(cmd_string)
 
     return QuizQuestion(
         id=question_id,
@@ -1113,14 +1101,19 @@ def generate_spot_difference_quiz(
 
     # Build the correct explanation of difference
     differences = []
-    if only_in_1:
-        for flag in only_in_1:
-            desc = _get_flag_description(base_cmd, flag)
-            differences.append(f"Command 1 has `{flag}` ({desc or 'unknown'})")
-    if only_in_2:
-        for flag in only_in_2:
+    has_unknown = False
+    for flag_set, label in [(only_in_1, "Command 1"), (only_in_2, "Command 2")]:
+        for flag in flag_set:
             desc = _get_flag_description(base_cmd, flag)
-            differences.append(f"Command 2 has `{flag}` ({desc or 'unknown'})")
+            # Handle numeric flags like -3 (shorthand for -n 3)
+            if not desc and re.match(r'^-\d+$', flag):
+                desc = f"Specify count ({flag[1:]})"
+            if not desc:
+                has_unknown = True
+            differences.append(f"{label} has `{flag}` ({desc or 'specifies an option'})")
+    # Skip questions where we can't explain the flags well
+    if has_unknown:
+        return None
     if parsed1["args"] != parsed2["args"]:
         differences.append(f"Different arguments: '{' '.join(parsed1['args'])}' vs '{' '.join(parsed2['args'])}'")
 
@@ -1236,14 +1229,33 @@ def generate_quiz_set(
     """
     questions: list[QuizQuestion] = []
 
+    # Filter out non-bash entries (Python code fragments, junk tokens, single chars)
+    junk_tokens = {'version', 'total', 'package', 'success', 'error', 'reading',
+                   'editing', 'done', 'warning', 'info', 'note', 'output',
+                   'task', 'goal', 'purpose', 'what', 'description'}
+    clean_commands = []
+    for cmd in analyzed_commands:
+        base = cmd.get("base_command", "")
+        if not base or len(base) < 2:
+            continue
+        if any(c in base for c in ('(', ')', '=', '{', '}')):
+            continue
+        if any(c in base for c in ('\\', '"', "'")) or '&' in base:
+            continue
+        if base[0].isupper() and base.isalpha() and base not in ('PATH', 'HOME'):
+            continue
+        if base.lower() in junk_tokens:
+            continue
+        clean_commands.append(cmd)
+
     # Filter commands by complexity >= 2
     eligible_commands = [
-        cmd for cmd in analyzed_commands
+        cmd for cmd in clean_commands
         if cmd.get("complexity", 0) >= 2
     ]
 
     if not eligible_commands:
-        eligible_commands = analyzed_commands
+        eligible_commands = clean_commands if clean_commands else analyzed_commands
 
     # Weight toward high-frequency commands
     weighted_commands = []
@@ -1270,12 +1282,17 @@ def generate_quiz_set(
         QuizType.SPOT_DIFFERENCE: set(),
     }
 
+    # Max command length for readable quiz questions
+    MAX_QUIZ_CMD_LEN = 200
+
     # Generate "What does this do?" questions
     random.shuffle(weighted_commands)
     for cmd in weighted_commands:
         if len([q for q in questions if q.quiz_type == QuizType.WHAT_DOES]) >= target_what_does:
             break
         cmd_id = cmd.get("command", "")
+        if len(cmd_id) > MAX_QUIZ_CMD_LEN:
+            continue
         if cmd_id not in used_per_type[QuizType.WHAT_DOES]:
             q = generate_what_does_quiz(cmd)
             questions.append(q)
@@ -1299,6 +1316,8 @@ def generate_quiz_set(
         if len([q for q in questions if q.quiz_type == QuizType.BUILD_COMMAND]) >= target_build:
             break
         cmd_id = cmd.get("command", "")
+        if len(cmd_id) > MAX_QUIZ_CMD_LEN:
+            continue
         if cmd_id not in used_per_type[QuizType.BUILD_COMMAND]:
             q = generate_build_command_quiz(cmd)
             questions.append(q)
@@ -1310,6 +1329,8 @@ def generate_quiz_set(
         if len([q for q in questions if q.quiz_type == QuizType.SPOT_DIFFERENCE]) >= target_spot_diff:
             break
         cmd_id = cmd.get("command", "")
+        if len(cmd_id) > MAX_QUIZ_CMD_LEN:
+            continue
         if cmd_id not in used_per_type[QuizType.SPOT_DIFFERENCE]:
             variant = _create_similar_command_variant(cmd)
             if variant:
@@ -1318,6 +1339,17 @@ def generate_quiz_set(
                     questions.append(q)
                     used_per_type[QuizType.SPOT_DIFFERENCE].add(cmd_id)
 
+    # Deduplicate by question text (same question can come from different commands)
+    seen_texts = set()
+    deduped = []
+    for q in questions:
+        # Normalize: take first 80 chars of question text
+        q_key = q.question_text[:80]
+        if q_key not in seen_texts:
+            deduped.append(q)
+            seen_texts.add(q_key)
+    questions = deduped
+
     # Shuffle final questions
     random.shuffle(questions)