@saulwade/swl-ses 1.0.0

package/scripts/vendor/claude-usage/scanner.py

@@ -0,0 +1,467 @@
+"""
+scanner.py - Scans Claude Code JSONL transcript files and stores data in SQLite.
+"""
+
+import json
+import os
+import glob
+import sqlite3
+from pathlib import Path
+from datetime import datetime, timezone
+
+PROJECTS_DIR = Path.home() / ".claude" / "projects"
+DB_PATH = Path.home() / ".claude" / "usage.db"
+
+
+def get_db(db_path=DB_PATH):
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def init_db(conn):
+    conn.executescript("""
+        CREATE TABLE IF NOT EXISTS sessions (
+            session_id      TEXT PRIMARY KEY,
+            project_name    TEXT,
+            first_timestamp TEXT,
+            last_timestamp  TEXT,
+            git_branch      TEXT,
+            total_input_tokens      INTEGER DEFAULT 0,
+            total_output_tokens     INTEGER DEFAULT 0,
+            total_cache_read        INTEGER DEFAULT 0,
+            total_cache_creation    INTEGER DEFAULT 0,
+            model           TEXT,
+            turn_count      INTEGER DEFAULT 0,
+            session_name    TEXT
+        );
+
+        CREATE TABLE IF NOT EXISTS turns (
+            id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id              TEXT,
+            timestamp               TEXT,
+            model                   TEXT,
+            input_tokens            INTEGER DEFAULT 0,
+            output_tokens           INTEGER DEFAULT 0,
+            cache_read_tokens       INTEGER DEFAULT 0,
+            cache_creation_tokens   INTEGER DEFAULT 0,
+            tool_name               TEXT,
+            cwd                     TEXT,
+            agent_name              TEXT,
+            skill_name              TEXT
+        );
+
+        CREATE TABLE IF NOT EXISTS processed_files (
+            path    TEXT PRIMARY KEY,
+            mtime   REAL,
+            lines   INTEGER
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id);
+        CREATE INDEX IF NOT EXISTS idx_turns_timestamp ON turns(timestamp);
+        CREATE INDEX IF NOT EXISTS idx_sessions_first ON sessions(first_timestamp);
+    """)
+    conn.commit()
+
+
+def migrate_db(conn):
+    """Agrega columnas nuevas a DBs existentes (idempotente)."""
+    turns_cols = {row[1] for row in conn.execute("PRAGMA table_info(turns)").fetchall()}
+    sessions_cols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()}
+    changed = False
+    for col in ("agent_name", "skill_name"):
+        if col not in turns_cols:
+            conn.execute(f"ALTER TABLE turns ADD COLUMN {col} TEXT")
+            changed = True
+    if "session_name" not in sessions_cols:
+        conn.execute("ALTER TABLE sessions ADD COLUMN session_name TEXT")
+        changed = True
+    if changed:
+        conn.commit()
+
+
+def project_name_from_cwd(cwd):
+    """Derive a friendly project name from cwd path."""
+    if not cwd:
+        return "unknown"
+    # Normalize to forward slashes, take last 2 components
+    parts = cwd.replace("\\", "/").rstrip("/").split("/")
+    if len(parts) >= 2:
+        return "/".join(parts[-2:])
+    return parts[-1] if parts else "unknown"
+
+
+def parse_jsonl_file(filepath):
+    """Parse a JSONL file and yield (session_data, turns) tuples."""
+    import re as _re
+    turns = []
+    session_meta = {}  # session_id -> dict
+
+    try:
+        with open(filepath, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+
+                rtype = record.get("type")
+                if rtype not in ("assistant", "user", "system"):
+                    continue
+
+                session_id = record.get("sessionId")
+                if not session_id:
+                    continue
+
+                timestamp = record.get("timestamp", "")
+                cwd = record.get("cwd", "")
+                git_branch = record.get("gitBranch", "")
+
+                # Update session metadata from any record
+                if session_id not in session_meta:
+                    session_meta[session_id] = {
+                        "session_id": session_id,
+                        "project_name": project_name_from_cwd(cwd),
+                        "first_timestamp": timestamp,
+                        "last_timestamp": timestamp,
+                        "git_branch": git_branch,
+                        "model": None,
+                        "session_name": None,
+                    }
+                else:
+                    meta = session_meta[session_id]
+                    if timestamp and (not meta["first_timestamp"] or timestamp < meta["first_timestamp"]):
+                        meta["first_timestamp"] = timestamp
+                    if timestamp and (not meta["last_timestamp"] or timestamp > meta["last_timestamp"]):
+                        meta["last_timestamp"] = timestamp
+                    if git_branch and not meta["git_branch"]:
+                        meta["git_branch"] = git_branch
+
+                # Extraer nombre de sesión desde comandos /rename
+                if rtype == "system" and record.get("subtype") == "local_command":
+                    content = record.get("content", "")
+                    if "Session renamed to:" in content:
+                        m = _re.search(r"Session renamed to:\s*(.+?)(?:</local-command-stdout>|$)", content)
+                        if m:
+                            session_meta[session_id]["session_name"] = m.group(1).strip()
+                    continue  # No generar turno para registros de sistema
+
+                if rtype == "assistant":
+                    msg = record.get("message", {})
+                    usage = msg.get("usage", {})
+                    model = msg.get("model", "")
+
+                    input_tokens = usage.get("input_tokens", 0) or 0
+                    output_tokens = usage.get("output_tokens", 0) or 0
+                    cache_read = usage.get("cache_read_input_tokens", 0) or 0
+                    cache_creation = usage.get("cache_creation_input_tokens", 0) or 0
+
+                    # Only record turns that have actual token usage
+                    if input_tokens + output_tokens + cache_read + cache_creation == 0:
+                        continue
+
+                    # Extract tool name, agent name and skill name from content
+                    tool_name = None
+                    agent_name = None
+                    skill_name = None
+                    for item in msg.get("content", []):
+                        if isinstance(item, dict) and item.get("type") == "tool_use":
+                            tool_name = item.get("name")
+                            tool_input = item.get("input", {}) or {}
+                            if tool_name == "Agent" and isinstance(tool_input, dict):
+                                agent_name = tool_input.get("subagent_type")
+                            elif tool_name == "Skill" and isinstance(tool_input, dict):
+                                skill_name = tool_input.get("skill")
+                            break
+
+                    if model:
+                        session_meta[session_id]["model"] = model
+
+                    turns.append({
+                        "session_id": session_id,
+                        "timestamp": timestamp,
+                        "model": model,
+                        "input_tokens": input_tokens,
+                        "output_tokens": output_tokens,
+                        "cache_read_tokens": cache_read,
+                        "cache_creation_tokens": cache_creation,
+                        "tool_name": tool_name,
+                        "cwd": cwd,
+                        "agent_name": agent_name,
+                        "skill_name": skill_name,
+                    })
+
+    except Exception as e:
+        print(f"  Warning: error reading {filepath}: {e}")
+
+    return list(session_meta.values()), turns
+
+
+def aggregate_sessions(session_metas, turns):
+    """Aggregate turn data back into session-level stats."""
+    from collections import defaultdict
+
+    session_stats = defaultdict(lambda: {
+        "total_input_tokens": 0,
+        "total_output_tokens": 0,
+        "total_cache_read": 0,
+        "total_cache_creation": 0,
+        "turn_count": 0,
+        "model": None,
+    })
+
+    for t in turns:
+        s = session_stats[t["session_id"]]
+        s["total_input_tokens"] += t["input_tokens"]
+        s["total_output_tokens"] += t["output_tokens"]
+        s["total_cache_read"] += t["cache_read_tokens"]
+        s["total_cache_creation"] += t["cache_creation_tokens"]
+        s["turn_count"] += 1
+        if t["model"]:
+            s["model"] = t["model"]
+
+    # Merge into session_metas
+    result = []
+    for meta in session_metas:
+        sid = meta["session_id"]
+        stats = session_stats[sid]
+        result.append({**meta, **stats})
+    return result
+
+
+def upsert_sessions(conn, sessions):
+    for s in sessions:
+        # Check if session exists
+        existing = conn.execute(
+            "SELECT total_input_tokens, total_output_tokens, total_cache_read, "
+            "total_cache_creation, turn_count FROM sessions WHERE session_id = ?",
+            (s["session_id"],)
+        ).fetchone()
+
+        if existing is None:
+            conn.execute("""
+                INSERT INTO sessions
+                    (session_id, project_name, first_timestamp, last_timestamp,
+                     git_branch, total_input_tokens, total_output_tokens,
+                     total_cache_read, total_cache_creation, model, turn_count,
+                     session_name)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                s["session_id"], s["project_name"], s["first_timestamp"],
+                s["last_timestamp"], s["git_branch"],
+                s["total_input_tokens"], s["total_output_tokens"],
+                s["total_cache_read"], s["total_cache_creation"],
+                s["model"], s["turn_count"],
+                s.get("session_name"),
+            ))
+        else:
+            # Update: add new tokens on top of existing (since we only insert new turns)
+            conn.execute("""
+                UPDATE sessions SET
+                    last_timestamp = MAX(last_timestamp, ?),
+                    total_input_tokens = total_input_tokens + ?,
+                    total_output_tokens = total_output_tokens + ?,
+                    total_cache_read = total_cache_read + ?,
+                    total_cache_creation = total_cache_creation + ?,
+                    turn_count = turn_count + ?,
+                    model = COALESCE(?, model),
+                    session_name = COALESCE(?, session_name)
+                WHERE session_id = ?
+            """, (
+                s["last_timestamp"],
+                s["total_input_tokens"], s["total_output_tokens"],
+                s["total_cache_read"], s["total_cache_creation"],
+                s["turn_count"], s["model"],
+                s.get("session_name"),
+                s["session_id"]
+            ))
+
+
+def insert_turns(conn, turns):
+    conn.executemany("""
+        INSERT INTO turns
+            (session_id, timestamp, model, input_tokens, output_tokens,
+             cache_read_tokens, cache_creation_tokens, tool_name, cwd,
+             agent_name, skill_name)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """, [
+        (t["session_id"], t["timestamp"], t["model"],
+         t["input_tokens"], t["output_tokens"],
+         t["cache_read_tokens"], t["cache_creation_tokens"],
+         t["tool_name"], t["cwd"],
+         t.get("agent_name"), t.get("skill_name"))
+        for t in turns
+    ])
+
+
+def scan(projects_dir=PROJECTS_DIR, db_path=DB_PATH, verbose=True):
+    conn = get_db(db_path)
+    init_db(conn)
+    migrate_db(conn)
+
+    jsonl_files = glob.glob(str(projects_dir / "**" / "*.jsonl"), recursive=True)
+    jsonl_files.sort()
+
+    new_files = 0
+    updated_files = 0
+    skipped_files = 0
+    total_turns = 0
+    total_sessions = set()
+
+    for filepath in jsonl_files:
+        try:
+            mtime = os.path.getmtime(filepath)
+        except OSError:
+            continue
+
+        row = conn.execute(
+            "SELECT mtime, lines FROM processed_files WHERE path = ?",
+            (filepath,)
+        ).fetchone()
+
+        if row and abs(row["mtime"] - mtime) < 0.01:
+            skipped_files += 1
+            continue
+
+        is_new = row is None
+        if verbose:
+            status = "NEW" if is_new else "UPD"
+            print(f"  [{status}] {os.path.relpath(filepath, projects_dir)}")
+
+        session_metas, turns = parse_jsonl_file(filepath)
+
+        if turns or session_metas:
+            sessions = aggregate_sessions(session_metas, turns)
+
+            # For incremental updates: only insert turns not already in DB
+            if not is_new:
+                # Get existing turns count to detect which are new
+                # Simple approach: delete old turns for these sessions and re-insert
+                # More correct: track file line count and only process new lines
+                old_lines = row["lines"] if row else 0
+                # Re-parse only if file grew
+                current_lines = sum(1 for _ in open(filepath, encoding="utf-8", errors="replace"))
+
+                if current_lines <= old_lines:
+                    conn.execute("UPDATE processed_files SET mtime = ? WHERE path = ?",
+                                 (mtime, filepath))
+                    conn.commit()
+                    skipped_files += 1
+                    continue
+
+                # Only process the new lines
+                new_turns = []
+                new_metas = {}
+                try:
+                    with open(filepath, encoding="utf-8", errors="replace") as f:
+                        for i, line in enumerate(f):
+                            if i < old_lines:
+                                continue
+                            line = line.strip()
+                            if not line:
+                                continue
+                            try:
+                                record = json.loads(line)
+                            except json.JSONDecodeError:
+                                continue
+
+                            rtype = record.get("type")
+                            if rtype != "assistant":
+                                continue
+
+                            session_id = record.get("sessionId")
+                            if not session_id:
+                                continue
+
+                            msg = record.get("message", {})
+                            usage = msg.get("usage", {})
+                            input_tokens = usage.get("input_tokens", 0) or 0
+                            output_tokens = usage.get("output_tokens", 0) or 0
+                            cache_read = usage.get("cache_read_input_tokens", 0) or 0
+                            cache_creation = usage.get("cache_creation_input_tokens", 0) or 0
+
+                            if input_tokens + output_tokens + cache_read + cache_creation == 0:
+                                continue
+
+                            tool_name = None
+                            agent_name = None
+                            skill_name = None
+                            for item in msg.get("content", []):
+                                if isinstance(item, dict) and item.get("type") == "tool_use":
+                                    tool_name = item.get("name")
+                                    tool_input = item.get("input", {}) or {}
+                                    if tool_name == "Agent" and isinstance(tool_input, dict):
+                                        agent_name = tool_input.get("subagent_type")
+                                    elif tool_name == "Skill" and isinstance(tool_input, dict):
+                                        skill_name = tool_input.get("skill")
+                                    break
+
+                            new_turns.append({
+                                "session_id": session_id,
+                                "timestamp": record.get("timestamp", ""),
+                                "model": msg.get("model", ""),
+                                "input_tokens": input_tokens,
+                                "output_tokens": output_tokens,
+                                "cache_read_tokens": cache_read,
+                                "cache_creation_tokens": cache_creation,
+                                "tool_name": tool_name,
+                                "cwd": record.get("cwd", ""),
+                                "agent_name": agent_name,
+                                "skill_name": skill_name,
+                            })
+                except Exception as e:
+                    print(f"  Warning: {e}")
+
+                turns = new_turns
+                sessions = aggregate_sessions(list(new_metas.values()) or [], turns)
+                # Update session timestamps from full parse
+                for meta in session_metas:
+                    sessions_to_update = [s for s in sessions if s["session_id"] == meta["session_id"]]
+                    if not sessions_to_update:
+                        # Session exists but no new turns -- still update timestamps
+                        sessions.append({**meta,
+                                         "total_input_tokens": 0,
+                                         "total_output_tokens": 0,
+                                         "total_cache_read": 0,
+                                         "total_cache_creation": 0,
+                                         "turn_count": 0,
+                                         "model": meta.get("model")})
+
+                updated_files += 1
+            else:
+                new_files += 1
+
+            upsert_sessions(conn, sessions)
+            insert_turns(conn, turns)
+
+            for s in sessions:
+                total_sessions.add(s["session_id"])
+            total_turns += len(turns)
+
+        # Record file as processed
+        line_count = sum(1 for _ in open(filepath, encoding="utf-8", errors="replace"))
+        conn.execute("""
+            INSERT OR REPLACE INTO processed_files (path, mtime, lines)
+            VALUES (?, ?, ?)
+        """, (filepath, mtime, line_count))
+        conn.commit()
+
+    if verbose:
+        print(f"\nScan complete:")
+        print(f"  New files:     {new_files}")
+        print(f"  Updated files: {updated_files}")
+        print(f"  Skipped files: {skipped_files}")
+        print(f"  Turns added:   {total_turns}")
+        print(f"  Sessions seen: {len(total_sessions)}")
+
+    conn.close()
+    return {"new": new_files, "updated": updated_files, "skipped": skipped_files,
+            "turns": total_turns, "sessions": len(total_sessions)}
+
+
+if __name__ == "__main__":
+    print(f"Scanning {PROJECTS_DIR} ...")
+    scan()

package/scripts/vendor/markitdown/cli.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""
+cli.py — Wrapper de MarkItDown para swl-ses
+
+Convierte cualquier archivo o URL soportado a Markdown limpio.
+Diseñado para ser invocado por agentes SWL vía Bash tool.
+
+Uso:
+    python cli.py <ruta_o_url>             → imprime Markdown en stdout
+    python cli.py <ruta_o_url> --title     → imprime título en stderr
+    python cli.py --check                  → verifica dependencias instaladas
+    python cli.py --formats                → lista formatos disponibles
+
+Códigos de salida:
+    0 → conversión exitosa
+    1 → formato no soportado (UnsupportedFormatException)
+    2 → dependencia faltante para ese formato (MissingDependencyException)
+    3 → error de conversión (FileConversionException)
+    4 → archivo no encontrado / URL inaccesible
+
+Para SWL agents — detectar éxito:
+    OUTPUT=$(python cli.py documento.xlsx 2>/dev/null) && [ -n "$OUTPUT" ]
+"""
+
+import sys
+import os
+
+# Compatibilidad con Windows paths en bash
+# Git bash convierte /d/... a D:\..., pero markitdown necesita Windows paths
+def normalize_path(path: str) -> str:
+    """Normaliza paths de bash para Windows si es necesario."""
+    if sys.platform == "win32" and path.startswith("/") and len(path) > 2 and path[2] == "/":
+        # Patrón /d/... → D:/...
+        drive = path[1].upper()
+        rest = path[2:]
+        return f"{drive}:{rest}"
+    return path
+
+
+def check_dependencies() -> dict:
+    """Verifica qué dependencias opcionales están instaladas."""
+    core = {}
+    optional = {}
+
+    # Core (requeridas)
+    for pkg in ["bs4", "requests", "markdownify", "magika", "charset_normalizer", "defusedxml"]:
+        try:
+            __import__(pkg)
+            core[pkg] = True
+        except ImportError:
+            core[pkg] = False
+
+    # Opcionales por formato
+    format_deps = {
+        "pdf": ["pdfminer", "pdfplumber"],
+        "docx": ["mammoth", "lxml"],
+        "xlsx/xls": ["pandas", "openpyxl"],
+        "pptx": ["pptx"],
+        "youtube": ["youtube_transcript_api"],
+    }
+
+    for fmt, pkgs in format_deps.items():
+        available = []
+        for pkg in pkgs:
+            try:
+                __import__(pkg)
+                available.append(pkg)
+            except ImportError:
+                pass
+        optional[fmt] = len(available) == len(pkgs)
+
+    return {"core": core, "formats": optional}
+
+
+def print_check():
+    """Imprime diagnóstico de dependencias instaladas."""
+    deps = check_dependencies()
+
+    all_core_ok = all(deps["core"].values())
+    print(f"markitdown — diagnóstico de dependencias")
+    print(f"Core: {'OK' if all_core_ok else 'INCOMPLETO'}")
+    for pkg, ok in deps["core"].items():
+        mark = "[OK]" if ok else "[--]"
+        print(f"  {mark} {pkg}")
+
+    print("\nFormatos opcionales:")
+    for fmt, ok in deps["formats"].items():
+        mark = "[OK]" if ok else "[--]"
+        print(f"  {mark} {fmt}")
+
+    if all_core_ok:
+        print("\nEstado: listo para conversión")
+        return 0
+    else:
+        print("\nEstado: dependencias core faltantes — ejecutar:")
+        print("  pip install markitdown[pdf,docx,pptx,xlsx]")
+        return 2
+
+
+def print_formats():
+    """Imprime formatos soportados según dependencias instaladas."""
+    deps = check_dependencies()
+
+    formats = [
+        ("Siempre disponibles", [
+            ".html / .htm", ".txt / .md", ".csv", ".json",
+            ".xml / .rss / .atom", ".zip (recursivo)", ".ipynb (Jupyter)",
+            ".epub", "URLs YouTube (transcripciones)", "URLs Wikipedia",
+        ]),
+        ("PDF (requiere: pdfminer, pdfplumber)", [".pdf"]),
+        ("Word (requiere: mammoth, lxml)", [".docx"]),
+        ("Excel (requiere: pandas, openpyxl)", [".xlsx", ".xls"]),
+        ("PowerPoint (requiere: python-pptx)", [".pptx"]),
+    ]
+
+    for categoria, fmts in formats:
+        print(f"\n{categoria}:")
+        for f in fmts:
+            print(f"  {f}")
+
+
+def main():
+    args = sys.argv[1:]
+
+    if not args or args[0] in ("-h", "--help"):
+        print(__doc__)
+        return 0
+
+    if args[0] == "--check":
+        return print_check()
+
+    if args[0] == "--formats":
+        print_formats()
+        return 0
+
+    show_title = "--title" in args
+    source_args = [a for a in args if not a.startswith("--")]
+
+    if not source_args:
+        print("Error: se requiere una ruta o URL", file=sys.stderr)
+        return 4
+
+    source = source_args[0]
+
+    # Normalizar path Windows si estamos en bash
+    if not source.startswith(("http://", "https://", "data:", "file:")):
+        source = normalize_path(source)
+        if not os.path.exists(source):
+            print(f"Error: archivo no encontrado: {source}", file=sys.stderr)
+            return 4
+
+    try:
+        from markitdown import (
+            MarkItDown,
+            UnsupportedFormatException,
+            FileConversionException,
+            MissingDependencyException,
+        )
+    except ImportError:
+        print("Error: markitdown no está instalado.", file=sys.stderr)
+        print("Ejecutar: pip install markitdown[pdf,docx,pptx,xlsx]", file=sys.stderr)
+        return 2
+
+    md = MarkItDown()
+
+    try:
+        result = md.convert(source)
+
+        if show_title and result.title:
+            print(f"TITLE: {result.title}", file=sys.stderr)
+
+        print(result.markdown)
+        return 0
+
+    except UnsupportedFormatException as e:
+        print(f"Formato no soportado: {e}", file=sys.stderr)
+        return 1
+
+    except MissingDependencyException as e:
+        print(f"Dependencia faltante: {e}", file=sys.stderr)
+        print("Ver: python cli.py --check", file=sys.stderr)
+        return 2
+
+    except FileConversionException as e:
+        print(f"Error de conversión: {e}", file=sys.stderr)
+        return 3
+
+    except Exception as e:
+        print(f"Error inesperado: {type(e).__name__}: {e}", file=sys.stderr)
+        return 3
+
+
+if __name__ == "__main__":
+    sys.exit(main())