oh-langfuse 0.1.25 → 0.1.26

package/bin/cli.js

@@ -712,15 +712,27 @@ async function interactiveMain(options) {
       rl,
       "What would you like to configure?",
       [
-        { label: "Setup Langfuse", value: "setup-langfuse", description: "Select one or more targets: Claude Code, OpenCode, Codex." },
-        { label: "Check Environment", value: "check-environment", description: "Verify required local tools before setup." },
-        { label: "Check Configuration", value: "check", description: "Inspect current setup without changing local files." },
-        { label: "Exit", value: "exit", description: "Close the setup console." }
+        { label: "Setup Langfuse", value: "setup-langfuse", description: "Select one or more targets: Claude Code, OpenCode, Codex." },
+        { label: "Update Installed Runtimes", value: "update", description: "Refresh installed Claude, OpenCode, and Codex Langfuse hooks/plugins." },
+        { label: "Check Environment", value: "check-environment", description: "Verify required local tools before setup." },
+        { label: "Check Configuration", value: "check", description: "Inspect current setup without changing local files." },
+        { label: "Exit", value: "exit", description: "Close the setup console." }
       ],
       options
     );
-
-    if (action === "setup-langfuse") return await setupLangfuseMenu(rl, options);
+
+    if (action === "setup-langfuse") return await setupLangfuseMenu(rl, options);
+    if (action === "update") {
+      const config = langfuseConfig(options.configOverrides);
+      return runNodeScript("update-langfuse-runtime.mjs", [
+        "all",
+        ...commonLangfuseArgs(config),
+        ...(hasValue(options.npmRegistry) ? [`--npmRegistry=${options.npmRegistry}`] : []),
+        ...(hasValue(options.pipIndexUrl) ? [`--pipIndexUrl=${options.pipIndexUrl}`] : []),
+        ...(options.skipCheck ? ["--skip-check"] : []),
+        ...(options.skipPluginInstall ? ["--skip-plugin-install"] : []),
+      ], options);
+    }
     if (action === "setup-claude") return await setupClaude(rl, options);
     if (action === "setup-opencode") return await setupOpenCode(rl, options);
     if (action === "setup-codex") return await setupCodex(rl, options);
@@ -758,7 +770,7 @@ async function setupLangfuseMenu(rl, options) {
   return code;
 }
 
-function printHelp() {
+function printHelp() {
   renderBrand({ dryRun: false });
   console.log("");
   renderSection("Usage", [
@@ -773,7 +785,12 @@ function printHelp() {
     "oh-langfuse check environment",
     "oh-langfuse check claude",
     "oh-langfuse check opencode",
-    "oh-langfuse check codex"
+    "oh-langfuse check codex",
+    "oh-langfuse update",
+    "oh-langfuse update all",
+    "oh-langfuse update claude",
+    "oh-langfuse update opencode",
+    "oh-langfuse update codex"
   ]);
   renderSection("Options", [
     `${paint("--dry-run", t.gold)}  Preview actions without writing files or installing packages.`,
@@ -821,8 +838,19 @@ async function main() {
     if (cmd === "setup" && target === "claude") return await setupClaude(rl, options);
     if (cmd === "setup" && target === "opencode") return await setupOpenCode(rl, options);
     if (cmd === "setup" && target === "codex") return await setupCodex(rl, options);
-    if (cmd === "setup") return await setupLangfuseMenu(rl, options);
-    if (cmd === "check" && target === "claude") return checkClaude(options);
+    if (cmd === "setup") return await setupLangfuseMenu(rl, options);
+    if (cmd === "update") {
+      const updateArgs = [
+        target || "all",
+        ...commonLangfuseArgs(langfuseConfig(options.configOverrides)),
+        ...(hasValue(options.npmRegistry) ? [`--npmRegistry=${options.npmRegistry}`] : []),
+        ...(hasValue(options.pipIndexUrl) ? [`--pipIndexUrl=${options.pipIndexUrl}`] : []),
+        ...(options.skipCheck ? ["--skip-check"] : []),
+        ...(options.skipPluginInstall ? ["--skip-plugin-install"] : []),
+      ];
+      return runNodeScript("update-langfuse-runtime.mjs", updateArgs, options);
+    }
+    if (cmd === "check" && target === "claude") return checkClaude(options);
     if (cmd === "check" && target === "opencode") return checkOpenCode(options);
     if (cmd === "check" && target === "codex") return checkCodex(options);
     if (cmd === "check" && target === "environment") {

package/codex_langfuse_notify.py

@@ -7,11 +7,12 @@ uses that signal to incrementally read the matching Codex session JSONL file and
 emit the new assistant/user/tool events to Langfuse.
 """
 
-import json
-import os
-import sys
-import time
-import hashlib
+import json
+import os
+import re
+import sys
+import time
+import hashlib
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
@@ -30,8 +31,9 @@ STATE_FILE = STATE_DIR / "state.json"
 LOCK_FILE = STATE_DIR / "state.lock"
 LOG_FILE = STATE_DIR / "codex_langfuse_notify.log"
 
-DEBUG = os.environ.get("CODEX_LANGFUSE_DEBUG", "").lower() == "true"
-MAX_CHARS = int(os.environ.get("CODEX_LANGFUSE_MAX_CHARS", "20000"))
+DEBUG = os.environ.get("CODEX_LANGFUSE_DEBUG", "").lower() == "true"
+MAX_CHARS = int(os.environ.get("CODEX_LANGFUSE_MAX_CHARS", "20000"))
+METRICS_SCHEMA_VERSION = "1.0"
 
 
 def log(level: str, message: str) -> None:
@@ -290,7 +292,7 @@ def extract_text(content: Any) -> str:
     return ""
 
 
-def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any]]:
+def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any]]:
     if not isinstance(value, str):
         try:
             text = json.dumps(value, ensure_ascii=False)
@@ -303,12 +305,163 @@ def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any
     if orig_len <= max_chars:
         return value if isinstance(value, str) else value, {"truncated": False, "orig_len": orig_len}
     kept = text[:max_chars]
-    return kept, {
-        "truncated": True,
-        "orig_len": orig_len,
-        "kept_len": len(kept),
-        "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
-    }
+    return kept, {
+        "truncated": True,
+        "orig_len": orig_len,
+        "kept_len": len(kept),
+        "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+    }
+
+
+def build_interaction_id(source: str, session_id: str, turn_number: int) -> str:
+    return f"{source or 'unknown'}:{session_id or 'unknown'}:{int(turn_number or 0)}"
+
+
+def _num_or_none(value: Any) -> Optional[int]:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int) and value >= 0:
+        return value
+    if isinstance(value, float) and value >= 0:
+        return int(value)
+    if isinstance(value, str):
+        try:
+            n = int(value)
+            return n if n >= 0 else None
+        except Exception:
+            return None
+    return None
+
+
+def _first_num(raw: Dict[str, Any], *keys: str) -> Optional[int]:
+    for key in keys:
+        if key in raw:
+            value = _num_or_none(raw.get(key))
+            if value is not None:
+                return value
+    return None
+
+
+def normalize_token_metrics(raw: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    if not isinstance(raw, dict) or not raw:
+        return {
+            "token_metrics_available": False,
+            "input_tokens": None,
+            "output_tokens": None,
+            "total_tokens": None,
+            "cache_read_tokens": None,
+            "reasoning_tokens": None,
+        }
+    input_tokens = _first_num(raw, "input", "input_tokens", "inputTokens")
+    output_tokens = _first_num(raw, "output", "output_tokens", "outputTokens")
+    total_tokens = _first_num(raw, "total", "total_tokens", "totalTokens")
+    if total_tokens is None and input_tokens is not None and output_tokens is not None:
+        total_tokens = input_tokens + output_tokens
+    cache_read_tokens = _first_num(raw, "cache_read_tokens", "cachedInputTokens", "cacheRead")
+    reasoning_tokens = _first_num(raw, "reasoning_tokens", "reasoningTokens", "reasoning")
+    available = any(v is not None for v in [input_tokens, output_tokens, total_tokens, cache_read_tokens, reasoning_tokens])
+    return {
+        "token_metrics_available": available,
+        "input_tokens": input_tokens if available else None,
+        "output_tokens": output_tokens if available else None,
+        "total_tokens": total_tokens if available else None,
+        "cache_read_tokens": cache_read_tokens if available else None,
+        "reasoning_tokens": reasoning_tokens if available else None,
+    }
+
+
+def _ratio(numerator: Optional[int], denominator: Optional[int]) -> Optional[float]:
+    if numerator is None or denominator in (None, 0):
+        return None
+    return numerator / denominator
+
+
+def build_interaction_metadata(
+    source: str,
+    user_id: Optional[str],
+    session_id: str,
+    turn_number: int,
+    token_metrics: Optional[Dict[str, Any]],
+    tool_call_count: int,
+    tool_result_count: int,
+    skill_use_count: int,
+    model: Optional[str],
+    user_message_count: int = 1,
+    assistant_message_count: int = 1,
+) -> Dict[str, Any]:
+    tokens = normalize_token_metrics(token_metrics)
+    return {
+        "source": source,
+        "user_id": user_id or "",
+        "session_id": session_id,
+        "interaction_id": build_interaction_id(source, session_id, turn_number),
+        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+        "interaction_count": 1,
+        "user_message_count": user_message_count,
+        "assistant_message_count": assistant_message_count,
+        "tool_call_count": int(tool_call_count or 0),
+        "tool_result_count": int(tool_result_count or 0),
+        "skill_use_count": int(skill_use_count or 0),
+        **tokens,
+        "model": model,
+        "turn_number": int(turn_number or 0),
+        "efficiency": {
+            "tokens_per_interaction": tokens.get("total_tokens"),
+            "tool_calls_per_interaction": int(tool_call_count or 0),
+            "skills_per_interaction": int(skill_use_count or 0),
+            "output_input_token_ratio": _ratio(tokens.get("output_tokens"), tokens.get("input_tokens")),
+            "tokens_per_tool_call": _ratio(tokens.get("total_tokens"), int(tool_call_count or 0)),
+        },
+    }
+
+
+def discover_known_skills(extra_roots: Optional[List[Path]] = None) -> set:
+    roots = [
+        CODEX_DIR / "skills",
+        Path.home() / ".claude" / "skills",
+        Path.home() / ".config" / "opencode" / "skill",
+    ]
+    if extra_roots:
+        roots.extend(extra_roots)
+    names = set()
+    for root in roots:
+        try:
+            if not root.exists():
+                continue
+            for skill_file in root.rglob("SKILL.md"):
+                names.add(skill_file.parent.name)
+        except Exception:
+            continue
+    return names
+
+
+def _skill_namespace(name: str) -> str:
+    return name.split(":", 1)[0] if ":" in name else ""
+
+
+def detect_skill_usages(tool_calls: List[Dict[str, Any]], known_skills: set) -> List[Dict[str, str]]:
+    found: Dict[str, str] = {}
+    for call in tool_calls or []:
+        tool_name = str(call.get("name") or "")
+        input_obj = call.get("input") if isinstance(call.get("input"), (dict, list, str)) else {}
+        if tool_name.lower() == "skill" and isinstance(input_obj, dict):
+            for key in ("skill_name", "skill", "name"):
+                value = input_obj.get(key)
+                if isinstance(value, str) and value.strip():
+                    found[value.strip()] = "tool_call"
+                    break
+        try:
+            text = json.dumps(input_obj, ensure_ascii=False)
+        except Exception:
+            text = str(input_obj)
+        for match in re.finditer(r"([A-Za-z]:)?[^\"'\n\r]*[\\/]+([^\\/\"'\n\r]+)[\\/]+SKILL\.md", text, re.IGNORECASE):
+            candidate = match.group(2)
+            if candidate and (candidate in known_skills or not known_skills):
+                found[candidate] = "skill_file_path"
+    return [
+        {"name": name, "skill_namespace": _skill_namespace(name), "detected_by": detected_by}
+        for name, detected_by in sorted(found.items())
+    ]
 
 
 def get_payload(row: Dict[str, Any]) -> Dict[str, Any]:
@@ -428,67 +581,133 @@ def emit_codex_turn(
 ) -> None:
     user_text, user_meta = truncate(material.get("user_text") or "")
     assistant_text, assistant_meta = truncate(material.get("assistant_text") or "")
-    usage_details = usage_details_from_codex(usage)
-    model = first_string(meta.get("model"), meta.get("model_provider")) or "codex"
-
-    with propagate_attributes(
-        user_id=user_id,
+    usage_details = usage_details_from_codex(usage)
+    model = first_string(meta.get("model"), meta.get("model_provider")) or "codex"
+    tool_calls = material.get("tool_calls") or []
+    tool_results = material.get("tool_results") or []
+    skill_usages = detect_skill_usages(tool_calls, discover_known_skills())
+    interaction_meta = build_interaction_metadata(
+        "codex",
+        user_id,
+        session_id,
+        turn_num,
+        usage_details,
+        len(tool_calls),
+        len(tool_results),
+        len(skill_usages),
+        model,
+        user_message_count=1 if material.get("user_text") else 0,
+        assistant_message_count=1 if material.get("assistant_text") else 0,
+    )
+    skill_summary = [
+        {"name": item["name"], "count": 1, "detected_by": item["detected_by"]}
+        for item in skill_usages
+    ]
+
+    with propagate_attributes(
+        user_id=user_id,
         session_id=session_id,
         trace_name=f"Codex - Turn {turn_num}",
         tags=["codex"],
     ):
         with langfuse.start_as_current_observation(
             name=f"Codex - Turn {turn_num}",
-            input={"role": "user", "content": user_text},
-            metadata={
-                "source": "codex",
-                "session_id": session_id,
-                "turn_number": turn_num,
-                "session_path": str(session_path),
+            input={"role": "user", "content": user_text},
+            metadata={
+                **interaction_meta,
+                "source": "codex",
+                "session_id": session_id,
+                "turn_number": turn_num,
+                "session_path": str(session_path),
                 "cwd": meta.get("cwd"),
                 "originator": meta.get("originator"),
-                "cli_version": meta.get("cli_version"),
-                "user_text": user_meta,
-                "usage": usage,
-            },
-        ) as trace_span:
-            with langfuse.start_as_current_observation(
-                name="Codex Response",
-                as_type="generation",
+                "cli_version": meta.get("cli_version"),
+                "user_text": user_meta,
+                "usage": usage,
+                "skills": skill_summary,
+            },
+        ) as trace_span:
+            with langfuse.start_as_current_observation(
+                name="AI Interaction",
+                input={"role": "user", "content": user_text},
+                output={"role": "assistant", "content": assistant_text},
+                metadata=interaction_meta,
+            ):
+                pass
+
+            with langfuse.start_as_current_observation(
+                name="Codex Response",
+                as_type="generation",
                 model=model,
                 input={"role": "user", "content": user_text},
-                output={"role": "assistant", "content": assistant_text},
-                usage_details=usage_details or None,
-                metadata={"assistant_text": assistant_meta},
-            ):
-                pass
-
-            for call in material.get("tool_calls") or []:
-                tool_input, input_meta = truncate(call.get("input"))
-                with langfuse.start_as_current_observation(
-                    name=f"Tool: {call.get('name') or 'tool'}",
-                    as_type="tool",
-                    input=tool_input,
-                    metadata={
-                        "tool_id": call.get("id"),
-                        "tool_name": call.get("name"),
-                        "input_meta": input_meta,
-                    },
-                ):
-                    pass
-
-            for result in material.get("tool_results") or []:
-                output, output_meta = truncate(result.get("output"))
-                with langfuse.start_as_current_observation(
-                    name=f"Tool Result: {result.get('name') or 'tool'}",
+                output={"role": "assistant", "content": assistant_text},
+                usage_details=usage_details or None,
+                metadata={
+                    "assistant_text": assistant_meta,
+                    "source": "codex",
+                    "user_id": user_id or "",
+                    "session_id": session_id,
+                    "interaction_id": interaction_meta["interaction_id"],
+                    "turn_number": turn_num,
+                },
+            ):
+                pass
+
+            for skill in skill_usages:
+                with langfuse.start_as_current_observation(
+                    name=f"Skill Use: {skill['name']}",
+                    metadata={
+                        "source": "codex",
+                        "user_id": user_id or "",
+                        "session_id": session_id,
+                        "interaction_id": interaction_meta["interaction_id"],
+                        "skill_name": skill["name"],
+                        "skill_namespace": skill["skill_namespace"],
+                        "detected_by": skill["detected_by"],
+                        "turn_number": turn_num,
+                        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+                    },
+                ):
+                    pass
+
+            for call in tool_calls:
+                tool_input, input_meta = truncate(call.get("input"))
+                with langfuse.start_as_current_observation(
+                    name=f"Tool: {call.get('name') or 'tool'}",
                     as_type="tool",
-                    metadata={
-                        "tool_id": result.get("id"),
-                        "tool_name": result.get("name"),
-                        "output_meta": output_meta,
-                    },
-                ) as tool_obs:
-                    tool_obs.update(output=output)
+                    input=tool_input,
+                    metadata={
+                        "source": "codex",
+                        "user_id": user_id or "",
+                        "session_id": session_id,
+                        "interaction_id": interaction_meta["interaction_id"],
+                        "tool_id": call.get("id"),
+                        "tool_name": call.get("name"),
+                        "turn_number": turn_num,
+                        "input_meta": input_meta,
+                        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+                    },
+                ):
+                    pass
+
+            for result in tool_results:
+                output, output_meta = truncate(result.get("output"))
+                with langfuse.start_as_current_observation(
+                    name=f"Tool Result: {result.get('name') or 'tool'}",
+                    as_type="tool",
+                    metadata={
+                        "source": "codex",
+                        "user_id": user_id or "",
+                        "session_id": session_id,
+                        "interaction_id": interaction_meta["interaction_id"],
+                        "tool_id": result.get("id"),
+                        "tool_name": result.get("name"),
+                        "turn_number": turn_num,
+                        "output_meta": output_meta,
+                        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+                    },
+                ) as tool_obs:
+                    tool_obs.update(output=output)
 
             trace_span.update(output={"role": "assistant", "content": assistant_text})