codex-skill-analytics 0.1.0

package/src/codex_skill_analytics/parser.py

@@ -0,0 +1,484 @@
+from __future__ import annotations
+
+import ast
+import hashlib
+import json
+import os
+import re
+import shlex
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+SKILL_BLOCK_RE = re.compile(r"<skill>.*?</skill>", re.DOTALL)
+SKILL_NAME_RE = re.compile(r"<name>([^<]+)</name>")
+SKILL_PATH_RE = re.compile(r"<path>([^<]+)</path>")
+ROOT_ALIAS_RE = re.compile(r"`(r\d+)`\s*=\s*`([^`]+)`")
+CATALOG_ENTRY_RE = re.compile(
+    r"^-\s+(.+?):\s+.*?\(file:\s*([^)]+/SKILL\.md)\)", re.MULTILINE
+)
+FRONTMATTER_NAME_RE = re.compile(r"^name:\s*['\"]?([^'\"\n]+)", re.MULTILINE)
+
+READERS = frozenset(
+    {
+        "cat",
+        "head",
+        "tail",
+        "sed",
+        "nl",
+        "less",
+        "more",
+        "bat",
+        "grep",
+        "rg",
+        "awk",
+        "cut",
+        "wc",
+    }
+)
+RUNNERS = frozenset({"python", "python3", "bash", "zsh", "sh", "node", "deno", "ruby", "perl", "pwsh"})
+SCRIPT_EXTENSIONS = (".py", ".sh", ".js", ".ts", ".rb", ".pl", ".ps1")
+
+
+@dataclass(frozen=True)
+class SkillRef:
+    name: str
+    path: str
+    scope: str
+
+    @property
+    def key(self) -> str:
+        return self.path
+
+    @property
+    def identity(self) -> str:
+        return f"{self.scope}:{self.name}"
+
+
+@dataclass(frozen=True)
+class SessionMeta:
+    thread_id: str
+    started_at: str
+    cwd: str
+    cli_version: str
+    parent_thread_id: str | None
+
+
+@dataclass(frozen=True)
+class Invocation:
+    thread_id: str
+    turn_id: str
+    timestamp: str
+    skill: SkillRef
+    invoke_type: str
+    evidence_type: str
+    source_path: str
+    source_line: int
+    source_order: int
+    call_id: str | None
+
+    @property
+    def invocation_id(self) -> str:
+        material = f"{self.thread_id}\0{self.turn_id}\0{self.skill.identity}"
+        return hashlib.sha256(material.encode()).hexdigest()
+
+    @property
+    def access_id(self) -> str:
+        material = "\0".join(
+            (
+                self.thread_id,
+                self.turn_id,
+                self.call_id or self.timestamp,
+                self.skill.key,
+                self.evidence_type,
+            )
+        )
+        return hashlib.sha256(material.encode()).hexdigest()
+
+
+def infer_scope(path: str, home: Path) -> str:
+    normalized = path.replace("\\", "/")
+    home_text = str(home).replace("\\", "/")
+    if "/plugins/cache/" in normalized:
+        return "plugin"
+    if normalized.startswith(f"{home_text}/.codex/skills/.system/"):
+        return "system"
+    if normalized.startswith(
+        (f"{home_text}/.agents/skills/", f"{home_text}/.codex/skills/")
+    ):
+        return "user"
+    if "/.agents/skills/" in normalized or "/.codex/skills/" in normalized:
+        return "repo"
+    return "unknown"
+
+
+def normalize_path(path: str | Path) -> str:
+    return os.path.realpath(os.path.abspath(os.path.expanduser(str(path))))
+
+
+def skill_name_from_file(path: Path) -> str:
+    try:
+        prefix = path.read_text(encoding="utf-8", errors="replace")[:8192]
+    except OSError:
+        return path.parent.name
+    match = FRONTMATTER_NAME_RE.search(prefix)
+    return match.group(1).strip() if match else path.parent.name
+
+
+def discover_installed_skills(home: Path | None = None) -> dict[str, SkillRef]:
+    home = (home or Path.home()).expanduser().resolve()
+    roots = (home / ".agents" / "skills", home / ".codex" / "skills", home / ".codex" / "plugins" / "cache")
+    result: dict[str, SkillRef] = {}
+    for root in roots:
+        if not root.is_dir():
+            continue
+        for path in root.rglob("SKILL.md"):
+            try:
+                canonical = normalize_path(path)
+            except OSError:
+                canonical = str(path.absolute())
+            result[canonical] = SkillRef(
+                name=skill_name_from_file(path),
+                path=canonical,
+                scope=infer_scope(canonical, home),
+            )
+    return result
+
+
+def iter_strings(value: Any) -> Iterator[str]:
+    if isinstance(value, str):
+        yield value
+    elif isinstance(value, list):
+        for item in value:
+            yield from iter_strings(item)
+    elif isinstance(value, dict):
+        for item in value.values():
+            yield from iter_strings(item)
+
+
+def catalog_skills_from_record(record: dict[str, Any], home: Path) -> list[SkillRef]:
+    found: list[SkillRef] = []
+    for text in iter_strings(record.get("payload")):
+        if "SKILL.md" not in text:
+            continue
+        aliases = {name: path for name, path in ROOT_ALIAS_RE.findall(text)}
+        for name, raw_path in CATALOG_ENTRY_RE.findall(text):
+            raw_path = raw_path.strip().strip("`")
+            alias, slash, suffix = raw_path.partition("/")
+            if slash and alias in aliases:
+                raw_path = str(Path(aliases[alias]) / suffix)
+            path = normalize_path(raw_path)
+            found.append(SkillRef(name=name.strip(), path=path, scope=infer_scope(path, home)))
+    return found
+
+
+def explicit_skills_from_record(record: dict[str, Any], home: Path) -> list[SkillRef]:
+    payload = record.get("payload")
+    if record.get("type") != "response_item" or not isinstance(payload, dict):
+        return []
+    if payload.get("type") != "message":
+        return []
+    found: list[SkillRef] = []
+    for text in iter_strings(payload.get("content")):
+        for block in SKILL_BLOCK_RE.findall(text):
+            name_match = SKILL_NAME_RE.search(block)
+            path_match = SKILL_PATH_RE.search(block)
+            if not name_match or not path_match:
+                continue
+            path = normalize_path(path_match.group(1).strip())
+            found.append(
+                SkillRef(name=name_match.group(1).strip(), path=path, scope=infer_scope(path, home))
+            )
+    return found
+
+
+def _skip_js_string(source: str, start: int) -> int:
+    quote = source[start]
+    i = start + 1
+    while i < len(source):
+        if source[i] == "\\":
+            i += 2
+            continue
+        if source[i] == quote:
+            return i + 1
+        i += 1
+    return len(source)
+
+
+def _iter_tool_objects(source: str, tool: str) -> Iterator[str]:
+    needle = f"tools.{tool}"
+    i = 0
+    while i < len(source):
+        char = source[i]
+        if char in "'\"`":
+            i = _skip_js_string(source, i)
+            continue
+        if source.startswith("//", i):
+            end = source.find("\n", i + 2)
+            i = len(source) if end < 0 else end + 1
+            continue
+        if source.startswith("/*", i):
+            end = source.find("*/", i + 2)
+            i = len(source) if end < 0 else end + 2
+            continue
+        if not source.startswith(needle, i):
+            i += 1
+            continue
+        open_paren = source.find("(", i + len(needle))
+        open_brace = source.find("{", open_paren + 1) if open_paren >= 0 else -1
+        if open_brace < 0:
+            i += len(needle)
+            continue
+        depth = 0
+        j = open_brace
+        while j < len(source):
+            if source[j] in "'\"`":
+                j = _skip_js_string(source, j)
+                continue
+            if source[j] == "{":
+                depth += 1
+            elif source[j] == "}":
+                depth -= 1
+                if depth == 0:
+                    yield source[open_brace : j + 1]
+                    i = j + 1
+                    break
+            j += 1
+        else:
+            return
+
+
+def _decode_js_string(source: str, start: int) -> tuple[str | None, int]:
+    if start >= len(source) or source[start] not in "'\"`":
+        return None, start
+    end = _skip_js_string(source, start)
+    if end <= start + 1 or end > len(source):
+        return None, end
+    token = source[start:end]
+    if token[0] == "`":
+        if "${" in token:
+            return None, end
+        body = token[1:-1]
+        return bytes(body, "utf-8").decode("unicode_escape"), end
+    try:
+        return ast.literal_eval(token), end
+    except (SyntaxError, ValueError):
+        if token[0] == '"':
+            try:
+                return json.loads(token), end
+            except json.JSONDecodeError:
+                pass
+    return None, end
+
+
+def _object_string_field(source: str, field: str) -> str | None:
+    i = 0
+    while i < len(source):
+        if source[i] in "'\"`":
+            i = _skip_js_string(source, i)
+            continue
+        match = re.match(r"[A-Za-z_$][A-Za-z0-9_$]*", source[i:])
+        if not match:
+            i += 1
+            continue
+        name = match.group(0)
+        i += len(name)
+        if name != field:
+            continue
+        while i < len(source) and source[i].isspace():
+            i += 1
+        if i >= len(source) or source[i] != ":":
+            continue
+        i += 1
+        while i < len(source) and source[i].isspace():
+            i += 1
+        value, _ = _decode_js_string(source, i)
+        return value
+    return None
+
+
+def exec_commands_from_input(source: str) -> Iterator[tuple[str, str | None]]:
+    for obj in _iter_tool_objects(source, "exec_command"):
+        command = _object_string_field(obj, "cmd")
+        if command is not None:
+            yield command, _object_string_field(obj, "workdir")
+
+
+def _shell_segments(command: str) -> list[list[str]]:
+    try:
+        lexer = shlex.shlex(command, posix=True, punctuation_chars="|&;()<>")
+        lexer.whitespace_split = True
+        lexer.commenters = ""
+        tokens = list(lexer)
+    except ValueError:
+        tokens = command.split()
+    segments: list[list[str]] = []
+    current: list[str] = []
+    for token in tokens:
+        if token and all(char in "|&;()<>" for char in token):
+            if current:
+                segments.append(current)
+                current = []
+        else:
+            current.append(token)
+    if current:
+        segments.append(current)
+    return segments
+
+
+def _executable_and_args(segment: list[str]) -> tuple[str, list[str]]:
+    index = 0
+    while index < len(segment) and ("=" in segment[index] and not segment[index].startswith("/")):
+        index += 1
+    while index < len(segment) and Path(segment[index]).name in {"sudo", "env", "command", "builtin"}:
+        index += 1
+        while index < len(segment) and segment[index].startswith("-"):
+            index += 1
+    if index >= len(segment):
+        return "", []
+    return Path(segment[index]).name.lower(), segment[index + 1 :]
+
+
+def _resolve_candidate(token: str, workdir: Path) -> str | None:
+    if not token or token.startswith("-") or "$" in token or "*" in token:
+        return None
+    token = token.rstrip(":,")
+    path = Path(os.path.expanduser(token))
+    if not path.is_absolute():
+        path = workdir / path
+    return normalize_path(path)
+
+
+def detect_skill_accesses(
+    command: str,
+    workdir: str | None,
+    catalog: dict[str, SkillRef],
+) -> list[tuple[SkillRef, str]]:
+    cwd = Path(workdir or os.getcwd()).expanduser()
+    found: list[tuple[SkillRef, str]] = []
+    seen: set[tuple[str, str]] = set()
+    script_dirs = [(str(Path(path).parent / "scripts"), skill) for path, skill in catalog.items()]
+    for segment in _shell_segments(command):
+        executable, args = _executable_and_args(segment)
+        if executable in READERS:
+            for token in args:
+                candidate = _resolve_candidate(token, cwd)
+                skill = catalog.get(candidate or "")
+                if skill is None:
+                    continue
+                key = (skill.key, "document")
+                if key not in seen:
+                    seen.add(key)
+                    found.append((skill, "document"))
+        runner = executable.removesuffix(".exe")
+        if runner not in RUNNERS:
+            continue
+        script_token = next(
+            (token for token in args if token != "--" and not token.startswith("-")), None
+        )
+        if script_token is None or not script_token.lower().endswith(SCRIPT_EXTENSIONS):
+            continue
+        candidate = _resolve_candidate(script_token, cwd)
+        if candidate is None:
+            continue
+        for scripts_dir, skill in script_dirs:
+            try:
+                inside = os.path.commonpath((candidate, scripts_dir)) == scripts_dir
+            except ValueError:
+                inside = False
+            key = (skill.key, "script")
+            if inside and key not in seen:
+                seen.add(key)
+                found.append((skill, "script"))
+                break
+    return found
+
+
+def parse_session_meta(record: dict[str, Any]) -> SessionMeta | None:
+    if record.get("type") != "session_meta" or not isinstance(record.get("payload"), dict):
+        return None
+    payload = record["payload"]
+    thread_id = str(payload.get("id") or payload.get("session_id") or "")
+    if not thread_id:
+        return None
+    return SessionMeta(
+        thread_id=thread_id,
+        started_at=str(payload.get("timestamp") or record.get("timestamp") or ""),
+        cwd=str(payload.get("cwd") or ""),
+        cli_version=str(payload.get("cli_version") or ""),
+        parent_thread_id=payload.get("parent_thread_id"),
+    )
+
+
+def parse_record_invocations(
+    record: dict[str, Any],
+    *,
+    session: SessionMeta,
+    catalog: dict[str, SkillRef],
+    source_path: str,
+    source_line: int,
+    home: Path,
+) -> list[Invocation]:
+    payload = record.get("payload")
+    if not isinstance(payload, dict):
+        return []
+    timestamp = str(record.get("timestamp") or "")
+    metadata = payload.get("internal_chat_message_metadata_passthrough")
+    metadata = metadata if isinstance(metadata, dict) else {}
+    turn_id = str(metadata.get("turn_id") or payload.get("turn_id") or "")
+    if not turn_id:
+        return []
+
+    invocations: list[Invocation] = []
+    source_order = 0
+    for skill in explicit_skills_from_record(record, home):
+        invocations.append(
+            Invocation(
+                thread_id=session.thread_id,
+                turn_id=turn_id,
+                timestamp=timestamp,
+                skill=skill,
+                invoke_type="explicit",
+                evidence_type="injected_skill_prompt",
+                source_path=source_path,
+                source_line=source_line,
+                source_order=source_order,
+                call_id=None,
+            )
+        )
+        source_order += 1
+
+    if record.get("type") != "response_item" or payload.get("type") != "custom_tool_call":
+        return invocations
+    if payload.get("name") != "exec" or not isinstance(payload.get("input"), str):
+        return invocations
+    call_id = str(payload.get("call_id") or payload.get("id") or "") or None
+    for command, workdir in exec_commands_from_input(payload["input"]):
+        for skill, access_type in detect_skill_accesses(command, workdir or session.cwd, catalog):
+            invocations.append(
+                Invocation(
+                    thread_id=session.thread_id,
+                    turn_id=turn_id,
+                    timestamp=timestamp,
+                    skill=skill,
+                    invoke_type="implicit",
+                    evidence_type=f"skill_{access_type}",
+                    source_path=source_path,
+                    source_line=source_line,
+                    source_order=source_order,
+                    call_id=call_id,
+                )
+            )
+            source_order += 1
+    return invocations
+
+
+def json_records(lines: Iterable[tuple[int, bytes]]) -> Iterator[tuple[int, dict[str, Any]]]:
+    for line_number, raw in lines:
+        try:
+            value = json.loads(raw)
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            continue
+        if isinstance(value, dict):
+            yield line_number, value

package/src/codex_skill_analytics/sync.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from .database import AnalyticsDB
+from .parser import (
+    SessionMeta,
+    catalog_skills_from_record,
+    discover_installed_skills,
+    parse_record_invocations,
+    parse_session_meta,
+)
+
+MAX_JSONL_LINE_BYTES = 16 * 1024 * 1024
+
+
+@dataclass
+class SyncStats:
+    files_seen: int = 0
+    files_read: int = 0
+    lines_read: int = 0
+    invocations_added: int = 0
+    accesses_added: int = 0
+    malformed_lines: int = 0
+    oversized_lines: int = 0
+
+
+def rollout_paths(codex_home: Path) -> Iterator[Path]:
+    for directory in (codex_home / "sessions", codex_home / "archived_sessions"):
+        if directory.is_dir():
+            yield from sorted(directory.rglob("*.jsonl"))
+
+
+def sync_history(codex_home: Path, db: AnalyticsDB) -> SyncStats:
+    codex_home = codex_home.expanduser().resolve()
+    home = codex_home.parent
+    installed_catalog = discover_installed_skills(home)
+    for skill in installed_catalog.values():
+        db.upsert_skill(skill)
+    stats = SyncStats()
+    for path in rollout_paths(codex_home):
+        catalog = dict(installed_catalog)
+        stats.files_seen += 1
+        source_path = str(path.resolve())
+        file_stat = path.stat()
+        checkpoint = db.checkpoint(source_path)
+        offset = int(checkpoint["byte_offset"]) if checkpoint else 0
+        line_number = int(checkpoint["line_number"]) if checkpoint else 0
+        if file_stat.st_size < offset:
+            offset = 0
+            line_number = 0
+        if file_stat.st_size == offset:
+            continue
+        session: SessionMeta | None = None
+        if offset > 0:
+            row = db.connection.execute(
+                "SELECT * FROM sessions WHERE source_path=?", (source_path,)
+            ).fetchone()
+            if row:
+                session = SessionMeta(
+                    thread_id=row["thread_id"],
+                    started_at=row["started_at"],
+                    cwd=row["cwd"],
+                    cli_version=row["cli_version"],
+                    parent_thread_id=row["parent_thread_id"],
+                )
+        next_offset = offset
+        next_line = line_number
+        file_had_lines = False
+        with path.open("rb") as handle:
+            handle.seek(offset)
+            while True:
+                line_start = handle.tell()
+                raw = handle.readline(MAX_JSONL_LINE_BYTES + 1)
+                if not raw:
+                    next_offset = handle.tell()
+                    break
+                if len(raw) > MAX_JSONL_LINE_BYTES:
+                    while raw and not raw.endswith(b"\n"):
+                        raw = handle.readline(MAX_JSONL_LINE_BYTES + 1)
+                    next_line += 1
+                    stats.lines_read += 1
+                    stats.oversized_lines += 1
+                    file_had_lines = True
+                    next_offset = handle.tell()
+                    continue
+                if not raw.endswith(b"\n"):
+                    next_offset = line_start
+                    break
+                next_line += 1
+                stats.lines_read += 1
+                file_had_lines = True
+                next_offset = handle.tell()
+                try:
+                    record = json.loads(raw)
+                except (UnicodeDecodeError, json.JSONDecodeError):
+                    stats.malformed_lines += 1
+                    continue
+                if not isinstance(record, dict):
+                    stats.malformed_lines += 1
+                    continue
+                parsed_meta = parse_session_meta(record)
+                if parsed_meta is not None:
+                    session = parsed_meta
+                    db.upsert_session(
+                        session,
+                        source_path,
+                        archived="archived_sessions" in path.parts,
+                    )
+                for skill in catalog_skills_from_record(record, home):
+                    catalog[skill.path] = skill
+                    db.upsert_skill(skill)
+                if session is None:
+                    continue
+                for invocation in parse_record_invocations(
+                    record,
+                    session=session,
+                    catalog=catalog,
+                    source_path=source_path,
+                    source_line=next_line,
+                    home=home,
+                ):
+                    invocation_added, access_added = db.add_invocation(invocation)
+                    stats.invocations_added += int(invocation_added)
+                    stats.accesses_added += int(access_added)
+        if not file_had_lines:
+            continue
+        stats.files_read += 1
+        db.save_checkpoint(
+            source_path,
+            byte_offset=next_offset,
+            line_number=next_line,
+            file_size=file_stat.st_size,
+            mtime_ns=file_stat.st_mtime_ns,
+        )
+        db.commit()
+    db.commit()
+    return stats