flonat-research 0.1.0

package/.scripts/skill-log-miner.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""
+Skill Log Miner — Extracts skill invocation data from session logs.
+
+Fallback observation method when the Skill tool doesn't emit hook events.
+Scans log/*.md files for `/skill-name` references, extracts structured
+invocation records, and writes to ~/.claude/ecc/observations-*.jsonl.
+
+Idempotent: tracks which log files have been mined in a watermark file.
+Re-running only processes new logs.
+
+Usage:
+    uv run python .scripts/skill-log-miner.py              # mine new logs
+    uv run python .scripts/skill-log-miner.py --backfill    # re-mine all logs
+    uv run python .scripts/skill-log-miner.py --dry-run     # show what would be mined
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+# ── Config ──────────────────────────────────────────────────────────────────
+ECC_DIR = Path.home() / ".claude" / "ecc"
+WATERMARK_FILE = ECC_DIR / "miner-watermark.json"
+
+# Where to find session logs — resolve from config
+def get_task_mgmt() -> Path | None:
+    cfg = Path.home() / ".config" / "task-mgmt" / "path"
+    if cfg.exists():
+        p = Path(cfg.read_text().strip())
+        if p.is_dir():
+            return p
+    return None
+
+# Also check project-specific log directories
+def find_log_dirs() -> list[Path]:
+    dirs = []
+    tm = get_task_mgmt()
+    if tm:
+        log_dir = tm / "log"
+        if log_dir.is_dir():
+            dirs.append(log_dir)
+    return dirs
+
+# Skill name pattern: `/skill-name` in backticks or at line start
+# Matches: `/proofread`, `/latex-autofix`, `/bib-validate`
+# Excludes: `/path/to/file`, `/dev/null`, `/usr/bin`, `/tmp/`
+SKILL_PATTERN = re.compile(
+    r'`/([a-z][a-z0-9-]+)`'          # backtick-quoted: `/proofread`
+    r'|(?:^|\s)/([a-z][a-z0-9-]+)'   # bare: /proofread at line start or after space
+    , re.MULTILINE
+)
+
+# Known non-skill patterns to exclude
+NON_SKILLS = {
+    "dev", "tmp", "usr", "bin", "etc", "var", "opt", "home",
+    "Users", "Library", "Applications", "System", "Volumes",
+    "clear", "help", "compact", "cost", "doctor", "init",
+    "login", "logout", "status", "version", "model",
+}
+
+# Date pattern from log filenames: YYYY-MM-DD-HHMM.md or YYYY-MM-DD-HHMM-compact.md
+LOG_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})-(\d{2})(\d{2})")
+
+# Project header in session logs
+PROJECT_RE = re.compile(r"^## Project:\s*(.+)", re.MULTILINE)
+
+
+def sha256_prefix(value: str, length: int = 12) -> str:
+    return hashlib.sha256(value.encode()).hexdigest()[:length]
+
+
+def load_known_skills() -> set[str]:
+    """Load valid skill names from ~/.claude/skills/."""
+    skills_dir = Path.home() / ".claude" / "skills"
+    if not skills_dir.is_dir():
+        return set()
+    return {d.name for d in skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()}
+
+
+def extract_skills_from_log(filepath: Path, known_skills: set[str]) -> list[dict]:
+    """Extract skill invocations from a session log file."""
+    try:
+        content = filepath.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+
+    # Skip compact logs (auto-generated, minimal skill data)
+    if "-compact" in filepath.name:
+        return []
+
+    # Extract date from filename
+    date_match = LOG_DATE_RE.search(filepath.stem)
+    if not date_match:
+        return []
+
+    date_str = date_match.group(1)
+    hour = date_match.group(2)
+    minute = date_match.group(3)
+    try:
+        log_date = datetime.strptime(f"{date_str}T{hour}:{minute}:00", "%Y-%m-%dT%H:%M:%S")
+        log_date = log_date.replace(tzinfo=timezone.utc)
+    except ValueError:
+        return []
+
+    # Extract project name
+    project_match = PROJECT_RE.search(content)
+    project_label = project_match.group(1).strip() if project_match else "unknown"
+
+    # Find all skill references
+    found_skills: dict[str, int] = {}
+    for match in SKILL_PATTERN.finditer(content):
+        skill_name = match.group(1) or match.group(2)
+        if not skill_name:
+            continue
+
+        # Filter out non-skills
+        if skill_name in NON_SKILLS:
+            continue
+        if len(skill_name) < 2:
+            continue
+
+        # Prefer known skills, but also accept plausible names
+        # (handles renamed/deleted skills in historical logs)
+        if skill_name in known_skills or len(skill_name) >= 3:
+            found_skills[skill_name] = found_skills.get(skill_name, 0) + 1
+
+    # Create events
+    events = []
+    for skill_name, mention_count in found_skills.items():
+        # Validate against known skills (strict mode for non-obvious names)
+        if skill_name not in known_skills and len(skill_name) < 5:
+            continue
+
+        event = {
+            "schema": "skill-event.v1",
+            "phase": "mined",
+            "skill": skill_name,
+            "session_hash": sha256_prefix(filepath.name),
+            "timestamp": log_date.isoformat(),
+            "project_label": project_label,
+            "project_hash": sha256_prefix(project_label),
+            "source_file": str(filepath.name),
+            "mention_count": mention_count,
+            "agent_context": "direct",
+        }
+        events.append(event)
+
+    return events
+
+
+def load_watermark() -> dict:
+    """Load the watermark file tracking which logs have been mined."""
+    if WATERMARK_FILE.exists():
+        try:
+            return json.loads(WATERMARK_FILE.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    return {"mined_files": [], "last_run": None}
+
+
+def save_watermark(watermark: dict):
+    """Save the watermark file."""
+    ECC_DIR.mkdir(parents=True, exist_ok=True)
+    os.chmod(ECC_DIR, 0o700)
+    old_umask = os.umask(0o077)
+    try:
+        WATERMARK_FILE.write_text(json.dumps(watermark, indent=2))
+    finally:
+        os.umask(old_umask)
+
+
+def write_events(events: list[dict]):
+    """Write events to daily JSONL files."""
+    if not events:
+        return
+
+    ECC_DIR.mkdir(parents=True, exist_ok=True)
+    os.chmod(ECC_DIR, 0o700)
+
+    # Group events by date (from their timestamp)
+    by_date: dict[str, list[dict]] = {}
+    for event in events:
+        try:
+            ts = datetime.fromisoformat(event["timestamp"])
+            date_key = ts.strftime("%Y-%m-%d")
+        except (KeyError, ValueError):
+            date_key = datetime.now().strftime("%Y-%m-%d")
+        by_date.setdefault(date_key, []).append(event)
+
+    old_umask = os.umask(0o077)
+    try:
+        for date_key, date_events in by_date.items():
+            filepath = ECC_DIR / f"observations-{date_key}.jsonl"
+            with open(filepath, "a") as f:
+                for event in date_events:
+                    f.write(json.dumps(event, separators=(",", ":")) + "\n")
+    finally:
+        os.umask(old_umask)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Mine session logs for skill invocations")
+    parser.add_argument("--backfill", action="store_true", help="Re-mine all logs (ignore watermark)")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would be mined without writing")
+    args = parser.parse_args()
+
+    known_skills = load_known_skills()
+    if not known_skills:
+        print("Warning: no skills found in ~/.claude/skills/", file=sys.stderr)
+
+    log_dirs = find_log_dirs()
+    if not log_dirs:
+        print("No log directories found.", file=sys.stderr)
+        sys.exit(1)
+
+    watermark = load_watermark()
+    mined_set = set(watermark.get("mined_files", [])) if not args.backfill else set()
+
+    # Collect all log files
+    all_logs = []
+    for log_dir in log_dirs:
+        for filepath in sorted(log_dir.glob("*.md")):
+            if filepath.name not in mined_set:
+                all_logs.append(filepath)
+
+    if not all_logs:
+        print("No new session logs to mine.")
+        return
+
+    # Mine each log
+    total_events = 0
+    total_skills = set()
+    all_events = []
+    newly_mined = []
+
+    for filepath in all_logs:
+        events = extract_skills_from_log(filepath, known_skills)
+        if events:
+            all_events.extend(events)
+            total_events += len(events)
+            for e in events:
+                total_skills.add(e["skill"])
+            if args.dry_run:
+                skills_in_log = [e["skill"] for e in events]
+                print(f"  {filepath.name}: {', '.join(skills_in_log)}")
+        newly_mined.append(filepath.name)
+
+    if args.dry_run:
+        print(f"\nDry run: {total_events} events from {len(newly_mined)} logs, "
+              f"{len(total_skills)} unique skills")
+        return
+
+    # Write events
+    write_events(all_events)
+
+    # Update watermark
+    mined_set.update(newly_mined)
+    watermark["mined_files"] = sorted(mined_set)
+    watermark["last_run"] = datetime.now(timezone.utc).isoformat()
+    save_watermark(watermark)
+
+    print(f"Mined {total_events} skill references from {len(newly_mined)} logs "
+          f"({len(total_skills)} unique skills)")
+
+
+if __name__ == "__main__":
+    main()

package/.scripts/sync-to-codex.sh

@@ -0,0 +1,252 @@
+#!/bin/bash
+# Sync Claude Code infrastructure to Codex CLI
+# Usage: bash .scripts/sync-to-codex.sh [--dry-run]
+#
+# Phases:
+#   1. Backup existing ~/.codex/ state
+#   2. Symlink skills from ~/.claude/skills/ → ~/.codex/skills/
+#   3. Generate ~/.codex/AGENTS.md from agents + rules
+#   4. Update ~/.codex/config.toml (model, sandbox, MCP servers)
+#   5. Validate
+
+set -euo pipefail
+
+DRY_RUN=false
+[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true
+
+CLAUDE_HOME="$HOME/.claude"
+CODEX_HOME="$HOME/.codex"
+SKILLS_SRC="$CLAUDE_HOME/skills"
+SKILLS_DST="$CODEX_HOME/skills"
+AGENTS_SRC="$CLAUDE_HOME/agents"
+RULES_SRC="$CLAUDE_HOME/rules"
+
+# Resolve Task Management root
+_CONFIG="$HOME/.config/task-mgmt/path"
+if [[ ! -f "$_CONFIG" ]]; then echo "Missing $_CONFIG" >&2; exit 1; fi
+TM="$(head -1 "$_CONFIG" | tr -d '\n')"
+
+# Colours
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+info()  { echo -e "${GREEN}✓${NC} $*"; }
+warn()  { echo -e "${YELLOW}⚠${NC} $*"; }
+error() { echo -e "${RED}✗${NC} $*" >&2; }
+
+# ── Phase 0: Pre-flight checks ──────────────────────────────────────────
+
+if [[ ! -d "$CODEX_HOME" ]]; then
+    error "Codex home directory not found at $CODEX_HOME"
+    exit 1
+fi
+
+if ! command -v codex &>/dev/null; then
+    error "Codex CLI not found in PATH"
+    exit 1
+fi
+
+if [[ ! -d "$SKILLS_SRC" ]]; then
+    error "Claude skills directory not found at $SKILLS_SRC"
+    exit 1
+fi
+
+echo "Syncing Claude Code → Codex CLI"
+echo "  Claude home: $CLAUDE_HOME"
+echo "  Codex home:  $CODEX_HOME"
+echo "  Dry run:     $DRY_RUN"
+echo ""
+
+# ── Phase 1: Backup ─────────────────────────────────────────────────────
+
+BACKUP_DIR="$CODEX_HOME/backups/$(date +%Y%m%d-%H%M%S)"
+if [[ "$DRY_RUN" == "false" ]]; then
+    mkdir -p "$BACKUP_DIR"
+    # Back up AGENTS.md and config.toml if they exist
+    [[ -f "$CODEX_HOME/AGENTS.md" ]] && cp "$CODEX_HOME/AGENTS.md" "$BACKUP_DIR/"
+    [[ -f "$CODEX_HOME/config.toml" ]] && cp "$CODEX_HOME/config.toml" "$BACKUP_DIR/"
+    info "Backup saved to $BACKUP_DIR"
+else
+    info "[dry-run] Would backup AGENTS.md and config.toml to $BACKUP_DIR"
+fi
+
+# ── Phase 2: Symlink skills ─────────────────────────────────────────────
+
+echo ""
+echo "Phase 2: Skills"
+
+mkdir -p "$SKILLS_DST"
+
+# Track counts
+LINKED=0
+SKIPPED=0
+REMOVED=0
+
+# Remove stale symlinks (point to skills that no longer exist in Claude)
+for link in "$SKILLS_DST"/*/; do
+    [[ ! -L "${link%/}" ]] && continue  # skip non-symlinks
+    link_name="$(basename "${link%/}")"
+    # Skip system skills
+    [[ "$link_name" == ".system" ]] && continue
+    # Check if symlink target still exists
+    if [[ ! -e "${link%/}" ]]; then
+        if [[ "$DRY_RUN" == "false" ]]; then
+            rm "$SKILLS_DST/$link_name"
+            ((REMOVED++))
+        else
+            warn "[dry-run] Would remove stale symlink: $link_name"
+            ((REMOVED++))
+        fi
+    fi
+done
+
+# Create symlinks for each Claude skill
+for skill_dir in "$SKILLS_SRC"/*/; do
+    [[ ! -d "$skill_dir" ]] && continue
+    skill_name="$(basename "$skill_dir")"
+
+    # Skip hidden directories
+    [[ "$skill_name" == .* ]] && continue
+
+    # Check for SKILL.md (required)
+    if [[ ! -f "$skill_dir/SKILL.md" ]]; then
+        warn "Skipping $skill_name (no SKILL.md)"
+        ((SKIPPED++))
+        continue
+    fi
+
+    # Check for collision with system skills
+    if [[ -d "$SKILLS_DST/.system/$skill_name" ]]; then
+        warn "Skipping $skill_name (conflicts with Codex system skill)"
+        ((SKIPPED++))
+        continue
+    fi
+
+    # Create or update symlink
+    if [[ "$DRY_RUN" == "false" ]]; then
+        ln -sfn "$skill_dir" "$SKILLS_DST/$skill_name"
+        ((LINKED++))
+    else
+        if [[ -L "$SKILLS_DST/$skill_name" ]]; then
+            ((LINKED++))
+        else
+            info "[dry-run] Would link: $skill_name"
+            ((LINKED++))
+        fi
+    fi
+done
+
+info "Skills: $LINKED linked, $SKIPPED skipped, $REMOVED stale removed"
+
+# ── Phase 3: Generate AGENTS.md ─────────────────────────────────────────
+
+echo ""
+echo "Phase 3: AGENTS.md"
+
+if [[ "$DRY_RUN" == "false" ]]; then
+    uv run python "$TM/.scripts/generate-codex-agents-md.py" \
+        --agents-dir "$AGENTS_SRC" \
+        --rules-dir "$RULES_SRC" \
+        --output "$CODEX_HOME/AGENTS.md"
+    info "Generated $CODEX_HOME/AGENTS.md"
+else
+    info "[dry-run] Would generate AGENTS.md from $AGENTS_SRC + $RULES_SRC"
+fi
+
+# ── Phase 4: Update config.toml ─────────────────────────────────────────
+
+echo ""
+echo "Phase 4: config.toml"
+
+CONFIG_FILE="$CODEX_HOME/config.toml"
+
+# Read existing config to preserve user settings (model, trust, migrations)
+# We only add/update the sections we manage; we don't overwrite the whole file.
+
+# Check if our managed sections already exist
+if grep -q "# --- Managed by sync-to-codex ---" "$CONFIG_FILE" 2>/dev/null; then
+    # Remove our managed block and re-append
+    if [[ "$DRY_RUN" == "false" ]]; then
+        sed -i '' '/^# --- Managed by sync-to-codex ---$/,/^# --- End managed ---$/d' "$CONFIG_FILE"
+    fi
+fi
+
+MANAGED_BLOCK=$(cat <<'TOML'
+# --- Managed by sync-to-codex ---
+# Do not edit this section manually; it is regenerated by sync-to-codex.sh
+
+[sandbox]
+# Default to workspace-write for interactive use
+permissions = ["disk-full-read-access"]
+
+[history]
+# Persist conversation history
+persistence = "save"
+
+# --- End managed ---
+TOML
+)
+
+if [[ "$DRY_RUN" == "false" ]]; then
+    echo "" >> "$CONFIG_FILE"
+    echo "$MANAGED_BLOCK" >> "$CONFIG_FILE"
+    info "Updated config.toml (managed block appended)"
+else
+    info "[dry-run] Would append managed block to config.toml"
+fi
+
+# ── Phase 5: Validation ─────────────────────────────────────────────────
+
+echo ""
+echo "Phase 5: Validation"
+
+ERRORS=0
+
+# Check skills count
+SKILL_COUNT=$(find "$SKILLS_DST" -maxdepth 1 -type l | wc -l | tr -d ' ')
+if [[ "$SKILL_COUNT" -gt 0 ]]; then
+    info "Skills directory: $SKILL_COUNT symlinks"
+else
+    error "No skill symlinks found"
+    ((ERRORS++))
+fi
+
+# Check AGENTS.md exists and is non-empty
+if [[ -f "$CODEX_HOME/AGENTS.md" && -s "$CODEX_HOME/AGENTS.md" ]]; then
+    AGENTS_LINES=$(wc -l < "$CODEX_HOME/AGENTS.md" | tr -d ' ')
+    info "AGENTS.md: $AGENTS_LINES lines"
+else
+    if [[ "$DRY_RUN" == "false" ]]; then
+        error "AGENTS.md missing or empty"
+        ((ERRORS++))
+    fi
+fi
+
+# Check config.toml is parseable (basic check)
+if [[ -f "$CONFIG_FILE" ]]; then
+    if python3 -c "
+import tomllib, sys
+with open('$CONFIG_FILE', 'rb') as f:
+    tomllib.load(f)
+" 2>/dev/null; then
+        info "config.toml: valid TOML"
+    else
+        error "config.toml: invalid TOML"
+        ((ERRORS++))
+    fi
+fi
+
+# Check Codex CLI version
+CODEX_VERSION=$(codex --version 2>/dev/null || echo "unknown")
+info "Codex CLI: $CODEX_VERSION"
+
+# Summary
+echo ""
+if [[ "$ERRORS" -eq 0 ]]; then
+    echo -e "${GREEN}Sync complete — no errors.${NC}"
+else
+    echo -e "${RED}Sync completed with $ERRORS error(s).${NC}"
+    exit 1
+fi