oh-langfuse 0.1.25 → 0.1.26

package/langfuse_hook.py

@@ -4,11 +4,12 @@ Claude Code -> Langfuse hook
 
 """
 
-import json
-import os
-import sys
-import time
-import hashlib
+import json
+import os
+import re
+import sys
+import time
+import hashlib
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
@@ -34,8 +35,9 @@ LOG_FILE = STATE_DIR / "langfuse_hook.log"
 STATE_FILE = STATE_DIR / "langfuse_state.json"
 LOCK_FILE = STATE_DIR / "langfuse_state.lock"
 
-DEBUG = os.environ.get("CC_LANGFUSE_DEBUG", "").lower() == "true"
-MAX_CHARS = int(os.environ.get("CC_LANGFUSE_MAX_CHARS", "20000"))
+DEBUG = os.environ.get("CC_LANGFUSE_DEBUG", "").lower() == "true"
+MAX_CHARS = int(os.environ.get("CC_LANGFUSE_MAX_CHARS", "20000"))
+METRICS_SCHEMA_VERSION = "1.0"
 
 # ----------------- Logging -----------------
 def _log(level: str, message: str) -> None:
@@ -237,14 +239,157 @@ def extract_text(content: Any) -> str:
         return "\n".join([p for p in parts if p])
     return ""
 
-def truncate_text(s: str, max_chars: int = MAX_CHARS) -> Tuple[str, Dict[str, Any]]:
+def truncate_text(s: str, max_chars: int = MAX_CHARS) -> Tuple[str, Dict[str, Any]]:
     if s is None:
         return "", {"truncated": False, "orig_len": 0}
     orig_len = len(s)
     if orig_len <= max_chars:
         return s, {"truncated": False, "orig_len": orig_len}
-    head = s[:max_chars]
-    return head, {"truncated": True, "orig_len": orig_len, "kept_len": len(head), "sha256": hashlib.sha256(s.encode("utf-8")).hexdigest()}
+    head = s[:max_chars]
+    return head, {"truncated": True, "orig_len": orig_len, "kept_len": len(head), "sha256": hashlib.sha256(s.encode("utf-8")).hexdigest()}
+
+def build_interaction_id(source: str, session_id: str, turn_number: int) -> str:
+    return f"{source or 'unknown'}:{session_id or 'unknown'}:{int(turn_number or 0)}"
+
+def _num_or_none(value: Any) -> Optional[int]:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int) and value >= 0:
+        return value
+    if isinstance(value, float) and value >= 0:
+        return int(value)
+    if isinstance(value, str):
+        try:
+            n = int(value)
+            return n if n >= 0 else None
+        except Exception:
+            return None
+    return None
+
+def _first_num(raw: Dict[str, Any], *keys: str) -> Optional[int]:
+    for key in keys:
+        if key in raw:
+            value = _num_or_none(raw.get(key))
+            if value is not None:
+                return value
+    return None
+
+def normalize_token_metrics(raw: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    if not isinstance(raw, dict) or not raw:
+        return {
+            "token_metrics_available": False,
+            "input_tokens": None,
+            "output_tokens": None,
+            "total_tokens": None,
+            "cache_read_tokens": None,
+            "reasoning_tokens": None,
+        }
+    input_tokens = _first_num(raw, "input", "input_tokens", "inputTokens")
+    output_tokens = _first_num(raw, "output", "output_tokens", "outputTokens")
+    total_tokens = _first_num(raw, "total", "total_tokens", "totalTokens")
+    if total_tokens is None and input_tokens is not None and output_tokens is not None:
+        total_tokens = input_tokens + output_tokens
+    cache_read_tokens = _first_num(raw, "cache_read_tokens", "cachedInputTokens", "cacheRead")
+    reasoning_tokens = _first_num(raw, "reasoning_tokens", "reasoningTokens", "reasoning")
+    available = any(v is not None for v in [input_tokens, output_tokens, total_tokens, cache_read_tokens, reasoning_tokens])
+    return {
+        "token_metrics_available": available,
+        "input_tokens": input_tokens if available else None,
+        "output_tokens": output_tokens if available else None,
+        "total_tokens": total_tokens if available else None,
+        "cache_read_tokens": cache_read_tokens if available else None,
+        "reasoning_tokens": reasoning_tokens if available else None,
+    }
+
+def _ratio(numerator: Optional[int], denominator: Optional[int]) -> Optional[float]:
+    if numerator is None or denominator in (None, 0):
+        return None
+    return numerator / denominator
+
+def build_interaction_metadata(
+    source: str,
+    user_id: Optional[str],
+    session_id: str,
+    turn_number: int,
+    token_metrics: Optional[Dict[str, Any]],
+    tool_call_count: int,
+    tool_result_count: int,
+    skill_use_count: int,
+    model: Optional[str],
+    user_message_count: int = 1,
+    assistant_message_count: int = 1,
+) -> Dict[str, Any]:
+    tokens = normalize_token_metrics(token_metrics)
+    interaction_id = build_interaction_id(source, session_id, turn_number)
+    return {
+        "source": source,
+        "user_id": user_id or "",
+        "session_id": session_id,
+        "interaction_id": interaction_id,
+        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+        "interaction_count": 1,
+        "user_message_count": user_message_count,
+        "assistant_message_count": assistant_message_count,
+        "tool_call_count": int(tool_call_count or 0),
+        "tool_result_count": int(tool_result_count or 0),
+        "skill_use_count": int(skill_use_count or 0),
+        **tokens,
+        "model": model,
+        "turn_number": int(turn_number or 0),
+        "efficiency": {
+            "tokens_per_interaction": tokens.get("total_tokens"),
+            "tool_calls_per_interaction": int(tool_call_count or 0),
+            "skills_per_interaction": int(skill_use_count or 0),
+            "output_input_token_ratio": _ratio(tokens.get("output_tokens"), tokens.get("input_tokens")),
+            "tokens_per_tool_call": _ratio(tokens.get("total_tokens"), int(tool_call_count or 0)),
+        },
+    }
+
+def discover_known_skills(extra_roots: Optional[List[Path]] = None) -> set:
+    roots = [
+        Path.home() / ".codex" / "skills",
+        Path.home() / ".claude" / "skills",
+        Path.home() / ".config" / "opencode" / "skill",
+    ]
+    if extra_roots:
+        roots.extend(extra_roots)
+    names = set()
+    for root in roots:
+        try:
+            if not root.exists():
+                continue
+            for skill_file in root.rglob("SKILL.md"):
+                names.add(skill_file.parent.name)
+        except Exception:
+            continue
+    return names
+
+def _skill_namespace(name: str) -> str:
+    return name.split(":", 1)[0] if ":" in name else ""
+
+def detect_skill_usages(tool_calls: List[Dict[str, Any]], known_skills: set) -> List[Dict[str, str]]:
+    found: Dict[str, str] = {}
+    for call in tool_calls or []:
+        tool_name = str(call.get("name") or "")
+        input_obj = call.get("input") if isinstance(call.get("input"), (dict, list, str)) else {}
+        if tool_name.lower() == "skill" and isinstance(input_obj, dict):
+            for key in ("skill_name", "skill", "name"):
+                value = input_obj.get(key)
+                if isinstance(value, str) and value.strip():
+                    found[value.strip()] = "tool_call"
+                    break
+        try:
+            text = json.dumps(input_obj, ensure_ascii=False)
+        except Exception:
+            text = str(input_obj)
+        for match in re.finditer(r"([A-Za-z]:)?[^\"'\n\r]*[\\/]+([^\\/\"'\n\r]+)[\\/]+SKILL\.md", text, re.IGNORECASE):
+            candidate = match.group(2)
+            if candidate and (candidate in known_skills or not known_skills):
+                found[candidate] = "skill_file_path"
+    return [
+        {"name": name, "skill_namespace": _skill_namespace(name), "detected_by": detected_by}
+        for name, detected_by in sorted(found.items())
+    ]
 
 def get_model(msg: Dict[str, Any]) -> str:
     m = msg.get("message")
@@ -461,13 +606,31 @@ def emit_turn(
     assistant_text, assistant_text_meta = truncate_text(assistant_text_raw)
 
     model = get_model(turn.assistant_msgs[0])
-    usage_details = get_usage(last_assistant)
-
-    tool_calls = _tool_calls_from_assistants(turn.assistant_msgs)
-
-    # attach tool outputs
-    for c in tool_calls:
-        if c["id"] and c["id"] in turn.tool_results_by_id:
+    usage_details = get_usage(last_assistant)
+
+    tool_calls = _tool_calls_from_assistants(turn.assistant_msgs)
+    skill_usages = detect_skill_usages(tool_calls, discover_known_skills())
+    interaction_meta = build_interaction_metadata(
+        "claude",
+        user_id,
+        session_id,
+        turn_num,
+        usage_details,
+        len(tool_calls),
+        len(turn.tool_results_by_id),
+        len(skill_usages),
+        model,
+        user_message_count=1,
+        assistant_message_count=len(turn.assistant_msgs),
+    )
+    skill_summary = [
+        {"name": item["name"], "count": 1, "detected_by": item["detected_by"]}
+        for item in skill_usages
+    ]
+
+    # attach tool outputs
+    for c in tool_calls:
+        if c["id"] and c["id"] in turn.tool_results_by_id:
             out_raw = turn.tool_results_by_id[c["id"]]
             out_str = out_raw if isinstance(out_raw, str) else json.dumps(out_raw, ensure_ascii=False)
             out_trunc, out_meta = truncate_text(out_str)
@@ -484,33 +647,65 @@ def emit_turn(
     ):
         with langfuse.start_as_current_observation(
             name=f"Claude Code - Turn {turn_num}",
-            input={"role": "user", "content": user_text},
-            metadata={
-                "source": "claude-code",
-                "session_id": session_id,
-                "turn_number": turn_num,
-                "transcript_path": str(transcript_path),
-                "user_text": user_text_meta,
-            },
-        ) as trace_span:
-            # LLM generation
-            with langfuse.start_as_current_observation(
-                name="Claude Response",
+            input={"role": "user", "content": user_text},
+            metadata={
+                **interaction_meta,
+                "source": "claude",
+                "session_id": session_id,
+                "turn_number": turn_num,
+                "transcript_path": str(transcript_path),
+                "user_text": user_text_meta,
+                "skills": skill_summary,
+            },
+        ) as trace_span:
+            with langfuse.start_as_current_observation(
+                name="AI Interaction",
+                input={"role": "user", "content": user_text},
+                output={"role": "assistant", "content": assistant_text},
+                metadata=interaction_meta,
+            ):
+                pass
+
+            # LLM generation
+            with langfuse.start_as_current_observation(
+                name="Claude Response",
                 as_type="generation",
                 model=model,
                 input={"role": "user", "content": user_text},
                 output={"role": "assistant", "content": assistant_text},
                 usage_details=usage_details or None,
                 metadata={
-                    "assistant_text": assistant_text_meta,
-                    "tool_count": len(tool_calls),
-                    "usage_details": usage_details,
-                },
-            ):
-                pass
-
-            # Tool observations
-            for tc in tool_calls:
+                    "assistant_text": assistant_text_meta,
+                    "tool_count": len(tool_calls),
+                    "usage_details": usage_details,
+                    "source": "claude",
+                    "user_id": user_id or "",
+                    "session_id": session_id,
+                    "interaction_id": interaction_meta["interaction_id"],
+                    "turn_number": turn_num,
+                },
+            ):
+                pass
+
+            for skill in skill_usages:
+                with langfuse.start_as_current_observation(
+                    name=f"Skill Use: {skill['name']}",
+                    metadata={
+                        "source": "claude",
+                        "user_id": user_id or "",
+                        "session_id": session_id,
+                        "interaction_id": interaction_meta["interaction_id"],
+                        "skill_name": skill["name"],
+                        "skill_namespace": skill["skill_namespace"],
+                        "detected_by": skill["detected_by"],
+                        "turn_number": turn_num,
+                        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+                    },
+                ):
+                    pass
+
+            # Tool observations
+            for tc in tool_calls:
                 in_obj = tc["input"]
                 # truncate tool input if it's a large string payload
                 if isinstance(in_obj, str):
@@ -522,14 +717,20 @@ def emit_turn(
                     name=f"Tool: {tc['name']}",
                     as_type="tool",
                     input=in_obj,
-                    metadata={
-                        "tool_name": tc["name"],
-                        "tool_id": tc["id"],
-                        "input_meta": in_meta,
-                        "output_meta": tc.get("output_meta"),
-                    },
-                ) as tool_obs:
-                    tool_obs.update(output=tc.get("output"))
+                    metadata={
+                        "source": "claude",
+                        "user_id": user_id or "",
+                        "session_id": session_id,
+                        "interaction_id": interaction_meta["interaction_id"],
+                        "tool_name": tc["name"],
+                        "tool_id": tc["id"],
+                        "turn_number": turn_num,
+                        "input_meta": in_meta,
+                        "output_meta": tc.get("output_meta"),
+                        "metrics_schema_version": METRICS_SCHEMA_VERSION,
+                    },
+                ) as tool_obs:
+                    tool_obs.update(output=tc.get("output"))
 
             trace_span.update(output={"role": "assistant", "content": assistant_text})
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-langfuse",
-  "version": "0.1.25",
+  "version": "0.1.26",
   "private": false,
   "type": "module",
   "description": "Use npm scripts to configure Claude Code / OpenCode / Codex with Langfuse tracing.",
@@ -21,8 +21,11 @@
     "scripts/opencode-langfuse-check.mjs",
     "scripts/opencode-langfuse-run.mjs",
     "scripts/opencode-langfuse-setup.mjs",
-    "scripts/resolve-opencode-cli.mjs",
+    "scripts/resolve-opencode-cli.mjs",
     "scripts/real-self-verify.mjs",
+    "scripts/metrics-utils.mjs",
+    "scripts/update-langfuse-runtime.mjs",
+    "scripts/update-utils.mjs",
     "langfuse_hook.py",
     "codex_langfuse_notify.py",
     "README.md",
@@ -31,9 +34,10 @@
     "setup-langfuse.bat",
     "setup-langfuse.sh"
   ],
-  "scripts": {
-    "start": "node bin/cli.js",
+  "scripts": {
+    "start": "node bin/cli.js",
     "check": "node --check bin/cli.js",
+    "test": "node --test tests/*.test.mjs",
     "pack:check": "npm pack --dry-run",
     "claude:setup": "node scripts/langfuse-setup.mjs",
     "claude:check": "node scripts/langfuse-check.mjs",
@@ -47,8 +51,9 @@
     "opencode:langfuse:run": "node scripts/opencode-langfuse-run.mjs",
     "codex:setup": "node scripts/codex-langfuse-setup.mjs",
     "codex:check": "node scripts/codex-langfuse-check.mjs",
-    "codex:langfuse:setup": "node scripts/codex-langfuse-setup.mjs",
+    "codex:langfuse:setup": "node scripts/codex-langfuse-setup.mjs",
     "codex:langfuse:check": "node scripts/codex-langfuse-check.mjs",
+    "update": "node scripts/update-langfuse-runtime.mjs",
     "self:verify": "node scripts/real-self-verify.mjs"
   },
   "dependencies": {}

package/scripts/metrics-utils.mjs

@@ -0,0 +1,126 @@
+export const METRICS_SCHEMA_VERSION = "1.0";
+
+function numberOrNull(value) {
+  if (typeof value === "string" && value.trim().startsWith("{")) {
+    try {
+      const parsed = JSON.parse(value);
+      return numberOrNull(parsed.intValue ?? parsed.doubleValue ?? parsed.value);
+    } catch {
+      return null;
+    }
+  }
+  const n = Number(value);
+  return Number.isFinite(n) && n >= 0 ? n : null;
+}
+
+export function buildInteractionId(source, sessionId, turnNumber) {
+  return `${String(source || "unknown")}:${String(sessionId || "unknown")}:${Number(turnNumber) || 0}`;
+}
+
+export function normalizeTokenMetrics(raw) {
+  if (!raw || typeof raw !== "object") {
+    return {
+      token_metrics_available: false,
+      input_tokens: null,
+      output_tokens: null,
+      total_tokens: null,
+      cache_read_tokens: null,
+      reasoning_tokens: null,
+    };
+  }
+
+  const input = numberOrNull(raw.input ?? raw.input_tokens ?? raw.inputTokens);
+  const output = numberOrNull(raw.output ?? raw.output_tokens ?? raw.outputTokens);
+  const total = numberOrNull(raw.total ?? raw.total_tokens ?? raw.totalTokens ?? (input != null && output != null ? input + output : null));
+  const cacheRead = numberOrNull(raw.cacheRead ?? raw.cache_read_tokens ?? raw.cachedInputTokens);
+  const reasoning = numberOrNull(raw.reasoning ?? raw.reasoning_tokens ?? raw.reasoningTokens);
+
+  const available = [input, output, total, cacheRead, reasoning].some((value) => value != null);
+  return {
+    token_metrics_available: available,
+    input_tokens: available ? input : null,
+    output_tokens: available ? output : null,
+    total_tokens: available ? total : null,
+    cache_read_tokens: available ? cacheRead : null,
+    reasoning_tokens: available ? reasoning : null,
+  };
+}
+
+function ratio(numerator, denominator) {
+  if (numerator == null || denominator == null || denominator === 0) return null;
+  return numerator / denominator;
+}
+
+export function buildInteractionMetadata(options = {}) {
+  const source = String(options.source || "unknown");
+  const sessionId = String(options.sessionId || options.session_id || "unknown");
+  const turnNumber = Number(options.turnNumber ?? options.turn_number ?? 0) || 0;
+  const tokenMetrics = normalizeTokenMetrics(options.tokenMetrics);
+  const toolCallCount = Number(options.toolCallCount ?? options.tool_call_count ?? 0) || 0;
+  const toolResultCount = Number(options.toolResultCount ?? options.tool_result_count ?? 0) || 0;
+  const skillUseCount = Number(options.skillUseCount ?? options.skill_use_count ?? 0) || 0;
+
+  return {
+    source,
+    user_id: String(options.userId || options.user_id || ""),
+    session_id: sessionId,
+    interaction_id: options.interactionId || buildInteractionId(source, sessionId, turnNumber),
+    metrics_schema_version: METRICS_SCHEMA_VERSION,
+    interaction_count: 1,
+    user_message_count: Number(options.userMessageCount ?? options.user_message_count ?? 1) || 1,
+    assistant_message_count: Number(options.assistantMessageCount ?? options.assistant_message_count ?? 1) || 1,
+    tool_call_count: toolCallCount,
+    tool_result_count: toolResultCount,
+    skill_use_count: skillUseCount,
+    ...tokenMetrics,
+    model: options.model || null,
+    turn_number: turnNumber,
+    efficiency: {
+      tokens_per_interaction: tokenMetrics.total_tokens,
+      tool_calls_per_interaction: toolCallCount,
+      skills_per_interaction: skillUseCount,
+      output_input_token_ratio: ratio(tokenMetrics.output_tokens, tokenMetrics.input_tokens),
+      tokens_per_tool_call: ratio(tokenMetrics.total_tokens, toolCallCount),
+    },
+  };
+}
+
+export function buildOpencodeMetricAttributes(options = {}) {
+  const attrs = options.attributes || {};
+  const userId = String(options.userId || attrs["oh.langfuse.user_id"] || attrs["langfuse.user.id"] || "");
+  const sessionId = String(attrs["ai.request.headers.x-opencode-session"] || options.sessionId || options.session_id || "unknown");
+  const requestId = String(attrs["ai.request.headers.x-opencode-request"] || options.requestId || options.request_id || "unknown");
+  const spanId = String(options.spanId || attrs["span.id"] || "unknown");
+  const provider = attrs["ai.model.provider"] || attrs["gen_ai.system"] || "";
+  const modelId = attrs["ai.model.id"] || attrs["gen_ai.request.model"] || attrs["ai.response.model"] || "";
+  const model = provider && modelId ? `${provider}/${modelId}` : provider || modelId || null;
+  const tokenMetrics = normalizeTokenMetrics({
+    input: attrs["ai.usage.inputTokens"] ?? attrs["ai.usage.promptTokens"] ?? attrs["gen_ai.usage.input_tokens"],
+    output: attrs["ai.usage.outputTokens"] ?? attrs["ai.usage.completionTokens"] ?? attrs["gen_ai.usage.output_tokens"],
+    total: attrs["ai.usage.totalTokens"],
+    cacheRead: attrs["ai.usage.cachedInputTokens"] ?? attrs["ai.usage.inputTokenDetails.cacheReadTokens"],
+    reasoning: attrs["ai.usage.reasoningTokens"] ?? attrs["ai.usage.outputTokenDetails.reasoningTokens"],
+  });
+
+  const out = {
+    "langfuse.observation.metadata.source": "opencode",
+    "langfuse.observation.metadata.user_id": userId,
+    "langfuse.observation.metadata.session_id": sessionId,
+    "langfuse.observation.metadata.interaction_id": `opencode:${userId || "unknown"}:${sessionId}:${requestId}:${spanId}`,
+    "langfuse.observation.metadata.metrics_schema_version": METRICS_SCHEMA_VERSION,
+    "langfuse.observation.metadata.interaction_count": 1,
+    "langfuse.observation.metadata.user_message_count": 1,
+    "langfuse.observation.metadata.assistant_message_count": 1,
+    "langfuse.observation.metadata.tool_call_count": 0,
+    "langfuse.observation.metadata.tool_result_count": 0,
+    "langfuse.observation.metadata.skill_use_count": 0,
+    "langfuse.observation.metadata.token_metrics_available": tokenMetrics.token_metrics_available,
+    "langfuse.observation.metadata.model": model,
+  };
+
+  for (const key of ["input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "reasoning_tokens"]) {
+    if (tokenMetrics[key] != null) out[`langfuse.observation.metadata.${key}`] = tokenMetrics[key];
+  }
+
+  return out;
+}