oh-langfuse 0.1.53 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -142
- package/bin/cli.js +425 -425
- package/codex_langfuse_notify.py +517 -517
- package/langfuse_hook.py +581 -581
- package/package.json +1 -1
- package/scripts/auto-update-runtime.mjs +190 -190
- package/scripts/codex-langfuse-check.mjs +81 -81
- package/scripts/codex-langfuse-setup.mjs +358 -314
- package/scripts/langfuse-check.mjs +180 -180
- package/scripts/langfuse-setup.mjs +370 -326
- package/scripts/log-filter-utils.mjs +26 -26
- package/scripts/metrics-utils.mjs +377 -377
- package/scripts/opencode-langfuse-check.mjs +9 -0
- package/scripts/opencode-langfuse-setup.mjs +944 -935
- package/scripts/real-self-verify.mjs +621 -621
- package/scripts/runtime-state-utils.mjs +53 -53
- package/scripts/update-langfuse-runtime.mjs +260 -260
- package/scripts/update-utils.mjs +73 -73
package/codex_langfuse_notify.py
CHANGED
|
@@ -7,46 +7,46 @@ uses that signal to incrementally read the matching Codex session JSONL file and
|
|
|
7
7
|
emit the new assistant/user/tool events to Langfuse.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
import json
|
|
11
|
-
import os
|
|
12
|
-
import re
|
|
13
|
-
import sys
|
|
14
|
-
import time
|
|
15
|
-
import hashlib
|
|
16
|
-
from dataclasses import dataclass
|
|
17
|
-
from datetime import datetime, timezone
|
|
18
|
-
from pathlib import Path
|
|
19
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
20
|
-
from urllib.parse import urlparse
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def configure_langfuse_no_proxy() -> None:
|
|
24
|
-
hosts = ["localhost", "127.0.0.1"]
|
|
25
|
-
for key in ("LANGFUSE_HOST", "LANGFUSE_BASEURL", "CODEX_LANGFUSE_BASE_URL"):
|
|
26
|
-
value = os.environ.get(key)
|
|
27
|
-
if not value:
|
|
28
|
-
continue
|
|
29
|
-
parsed = urlparse(value if "://" in value else f"http://{value}")
|
|
30
|
-
if parsed.hostname:
|
|
31
|
-
hosts.append(parsed.hostname)
|
|
32
|
-
if parsed.netloc:
|
|
33
|
-
hosts.append(parsed.netloc)
|
|
34
|
-
existing = []
|
|
35
|
-
for key in ("NO_PROXY", "no_proxy"):
|
|
36
|
-
existing.extend([item.strip() for item in os.environ.get(key, "").split(",") if item.strip()])
|
|
37
|
-
merged = []
|
|
38
|
-
for item in [*existing, *hosts]:
|
|
39
|
-
if item and item not in merged:
|
|
40
|
-
merged.append(item)
|
|
41
|
-
if merged:
|
|
42
|
-
value = ",".join(merged)
|
|
43
|
-
os.environ["NO_PROXY"] = value
|
|
44
|
-
os.environ["no_proxy"] = value
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
configure_langfuse_no_proxy()
|
|
48
|
-
|
|
49
|
-
try:
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
import hashlib
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
20
|
+
from urllib.parse import urlparse
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def configure_langfuse_no_proxy() -> None:
|
|
24
|
+
hosts = ["localhost", "127.0.0.1"]
|
|
25
|
+
for key in ("LANGFUSE_HOST", "LANGFUSE_BASEURL", "CODEX_LANGFUSE_BASE_URL"):
|
|
26
|
+
value = os.environ.get(key)
|
|
27
|
+
if not value:
|
|
28
|
+
continue
|
|
29
|
+
parsed = urlparse(value if "://" in value else f"http://{value}")
|
|
30
|
+
if parsed.hostname:
|
|
31
|
+
hosts.append(parsed.hostname)
|
|
32
|
+
if parsed.netloc:
|
|
33
|
+
hosts.append(parsed.netloc)
|
|
34
|
+
existing = []
|
|
35
|
+
for key in ("NO_PROXY", "no_proxy"):
|
|
36
|
+
existing.extend([item.strip() for item in os.environ.get(key, "").split(",") if item.strip()])
|
|
37
|
+
merged = []
|
|
38
|
+
for item in [*existing, *hosts]:
|
|
39
|
+
if item and item not in merged:
|
|
40
|
+
merged.append(item)
|
|
41
|
+
if merged:
|
|
42
|
+
value = ",".join(merged)
|
|
43
|
+
os.environ["NO_PROXY"] = value
|
|
44
|
+
os.environ["no_proxy"] = value
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
configure_langfuse_no_proxy()
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
50
|
from langfuse import Langfuse, propagate_attributes
|
|
51
51
|
except Exception:
|
|
52
52
|
sys.exit(0)
|
|
@@ -59,10 +59,10 @@ STATE_FILE = STATE_DIR / "state.json"
|
|
|
59
59
|
LOCK_FILE = STATE_DIR / "state.lock"
|
|
60
60
|
LOG_FILE = STATE_DIR / "codex_langfuse_notify.log"
|
|
61
61
|
|
|
62
|
-
DEBUG = os.environ.get("CODEX_LANGFUSE_DEBUG", "").lower() == "true"
|
|
63
|
-
MAX_CHARS = int(os.environ.get("CODEX_LANGFUSE_MAX_CHARS", "20000"))
|
|
64
|
-
METRICS_SCHEMA_VERSION = "1.1"
|
|
65
|
-
AGENT_NAME = "codex"
|
|
62
|
+
DEBUG = os.environ.get("CODEX_LANGFUSE_DEBUG", "").lower() == "true"
|
|
63
|
+
MAX_CHARS = int(os.environ.get("CODEX_LANGFUSE_MAX_CHARS", "20000"))
|
|
64
|
+
METRICS_SCHEMA_VERSION = "1.1"
|
|
65
|
+
AGENT_NAME = "codex"
|
|
66
66
|
|
|
67
67
|
|
|
68
68
|
def log(level: str, message: str) -> None:
|
|
@@ -321,7 +321,7 @@ def extract_text(content: Any) -> str:
|
|
|
321
321
|
return ""
|
|
322
322
|
|
|
323
323
|
|
|
324
|
-
def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any]]:
|
|
324
|
+
def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any]]:
|
|
325
325
|
if not isinstance(value, str):
|
|
326
326
|
try:
|
|
327
327
|
text = json.dumps(value, ensure_ascii=False)
|
|
@@ -334,363 +334,363 @@ def truncate(value: Any, max_chars: int = MAX_CHARS) -> Tuple[Any, Dict[str, Any
|
|
|
334
334
|
if orig_len <= max_chars:
|
|
335
335
|
return value if isinstance(value, str) else value, {"truncated": False, "orig_len": orig_len}
|
|
336
336
|
kept = text[:max_chars]
|
|
337
|
-
return kept, {
|
|
338
|
-
"truncated": True,
|
|
339
|
-
"orig_len": orig_len,
|
|
340
|
-
"kept_len": len(kept),
|
|
341
|
-
"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
def build_interaction_id(source: str, session_id: str, turn_number: int) -> str:
|
|
346
|
-
return f"{source or 'unknown'}:{session_id or 'unknown'}:{int(turn_number or 0)}"
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
def _num_or_none(value: Any) -> Optional[int]:
|
|
350
|
-
if isinstance(value, bool):
|
|
351
|
-
return None
|
|
352
|
-
if isinstance(value, int) and value >= 0:
|
|
353
|
-
return value
|
|
354
|
-
if isinstance(value, float) and value >= 0:
|
|
355
|
-
return int(value)
|
|
356
|
-
if isinstance(value, str):
|
|
357
|
-
try:
|
|
358
|
-
n = int(value)
|
|
359
|
-
return n if n >= 0 else None
|
|
360
|
-
except Exception:
|
|
361
|
-
return None
|
|
362
|
-
return None
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
def _first_num(raw: Dict[str, Any], *keys: str) -> Optional[int]:
|
|
366
|
-
for key in keys:
|
|
367
|
-
if key in raw:
|
|
368
|
-
value = _num_or_none(raw.get(key))
|
|
369
|
-
if value is not None:
|
|
370
|
-
return value
|
|
371
|
-
return None
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
def normalize_token_metrics(raw: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
375
|
-
if not isinstance(raw, dict) or not raw:
|
|
376
|
-
return {
|
|
377
|
-
"token_metrics_available": False,
|
|
378
|
-
"input_tokens": None,
|
|
379
|
-
"output_tokens": None,
|
|
380
|
-
"total_tokens": None,
|
|
381
|
-
"cache_read_tokens": None,
|
|
382
|
-
"reasoning_tokens": None,
|
|
383
|
-
}
|
|
384
|
-
input_tokens = _first_num(raw, "input", "input_tokens", "inputTokens")
|
|
385
|
-
output_tokens = _first_num(raw, "output", "output_tokens", "outputTokens")
|
|
386
|
-
total_tokens = _first_num(raw, "total", "total_tokens", "totalTokens")
|
|
387
|
-
if total_tokens is None and input_tokens is not None and output_tokens is not None:
|
|
388
|
-
total_tokens = input_tokens + output_tokens
|
|
389
|
-
cache_read_tokens = _first_num(raw, "cache_read_tokens", "cachedInputTokens", "cacheRead")
|
|
390
|
-
reasoning_tokens = _first_num(raw, "reasoning_tokens", "reasoningTokens", "reasoning")
|
|
391
|
-
available = any(v is not None for v in [input_tokens, output_tokens, total_tokens, cache_read_tokens, reasoning_tokens])
|
|
392
|
-
return {
|
|
393
|
-
"token_metrics_available": available,
|
|
394
|
-
"input_tokens": input_tokens if available else None,
|
|
395
|
-
"output_tokens": output_tokens if available else None,
|
|
396
|
-
"total_tokens": total_tokens if available else None,
|
|
397
|
-
"cache_read_tokens": cache_read_tokens if available else None,
|
|
398
|
-
"reasoning_tokens": reasoning_tokens if available else None,
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
def _ratio(numerator: Optional[int], denominator: Optional[int]) -> Optional[float]:
|
|
403
|
-
if numerator is None or denominator in (None, 0):
|
|
404
|
-
return None
|
|
405
|
-
return numerator / denominator
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
def build_interaction_metadata(
|
|
409
|
-
source: str,
|
|
410
|
-
user_id: Optional[str],
|
|
411
|
-
session_id: str,
|
|
412
|
-
turn_number: int,
|
|
413
|
-
token_metrics: Optional[Dict[str, Any]],
|
|
414
|
-
tool_call_count: int,
|
|
415
|
-
tool_result_count: int,
|
|
416
|
-
skill_use_count: int,
|
|
417
|
-
model: Optional[str],
|
|
418
|
-
user_message_count: int = 1,
|
|
419
|
-
assistant_message_count: int = 1,
|
|
420
|
-
skill_use_events: Optional[List[Dict[str, Any]]] = None,
|
|
421
|
-
) -> Dict[str, Any]:
|
|
422
|
-
tokens = normalize_token_metrics(token_metrics)
|
|
423
|
-
interaction_id = build_interaction_id(source, session_id, turn_number)
|
|
424
|
-
events = list(skill_use_events or [])
|
|
425
|
-
skill_names_all = [str(event.get("skill_name") or "") for event in events if event.get("skill_name")]
|
|
426
|
-
unique_skill_names = list(dict.fromkeys(skill_names_all))
|
|
427
|
-
skill_invocation_modes = [str(event.get("skill_invocation_mode") or "") for event in events if event.get("skill_invocation_mode")]
|
|
428
|
-
skill_agent_paths = [str(event.get("skill_agent_path") or "") for event in events if event.get("skill_agent_path")]
|
|
429
|
-
effective_skill_count = len(events) if events else int(skill_use_count or 0)
|
|
430
|
-
return {
|
|
431
|
-
"source": source,
|
|
432
|
-
"agent": source,
|
|
433
|
-
"user_id": user_id or "",
|
|
434
|
-
"session_id": session_id,
|
|
435
|
-
"interaction_id": interaction_id,
|
|
436
|
-
"metrics_schema_version": METRICS_SCHEMA_VERSION,
|
|
437
|
-
"interaction_count": 1,
|
|
438
|
-
"user_message_count": user_message_count,
|
|
439
|
-
"assistant_message_count": assistant_message_count,
|
|
440
|
-
"tool_call_count": int(tool_call_count or 0),
|
|
441
|
-
"tool_result_count": int(tool_result_count or 0),
|
|
442
|
-
"skill_use_count": effective_skill_count,
|
|
443
|
-
"unique_skill_count": len(unique_skill_names),
|
|
444
|
-
"repeated_skill_count": max(0, effective_skill_count - len(unique_skill_names)),
|
|
445
|
-
**tokens,
|
|
446
|
-
"model": model,
|
|
447
|
-
"turn_number": int(turn_number or 0),
|
|
448
|
-
"efficiency": {
|
|
449
|
-
"tokens_per_interaction": tokens.get("total_tokens"),
|
|
450
|
-
"tool_calls_per_interaction": int(tool_call_count or 0),
|
|
451
|
-
"skills_per_interaction": effective_skill_count,
|
|
452
|
-
"output_input_token_ratio": _ratio(tokens.get("output_tokens"), tokens.get("input_tokens")),
|
|
453
|
-
"tokens_per_tool_call": _ratio(tokens.get("total_tokens"), int(tool_call_count or 0)),
|
|
454
|
-
},
|
|
455
|
-
**({
|
|
456
|
-
"skill_names": unique_skill_names,
|
|
457
|
-
"skill_names_all": skill_names_all,
|
|
458
|
-
"skill_invocation_modes": skill_invocation_modes,
|
|
459
|
-
"skill_agent_paths": skill_agent_paths,
|
|
460
|
-
} if events else {}),
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
def discover_known_skills(extra_roots: Optional[List[Path]] = None) -> set:
|
|
465
|
-
roots = [
|
|
466
|
-
CODEX_DIR / "skills",
|
|
467
|
-
CODEX_DIR / "plugins" / "cache",
|
|
468
|
-
Path.home() / ".claude" / "skills",
|
|
469
|
-
Path.home() / ".config" / "opencode" / "skill",
|
|
470
|
-
]
|
|
471
|
-
if extra_roots:
|
|
472
|
-
roots.extend(extra_roots)
|
|
473
|
-
names = set()
|
|
474
|
-
for root in roots:
|
|
475
|
-
try:
|
|
476
|
-
if not root.exists():
|
|
477
|
-
continue
|
|
478
|
-
for skill_file in root.rglob("SKILL.md"):
|
|
479
|
-
names.add(skill_file.parent.name)
|
|
480
|
-
except Exception:
|
|
481
|
-
continue
|
|
482
|
-
return names
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
def _skill_namespace(name: str) -> str:
|
|
486
|
-
return name.split(":", 1)[0] if ":" in name else ""
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
def _skill_agent_from_interaction_id(interaction_id: str) -> str:
|
|
490
|
-
return str(interaction_id or "unknown").split(":", 1)[0] or "unknown"
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
def _skill_agent_path(agent: str, detected_by: str) -> str:
|
|
494
|
-
if agent == "codex":
|
|
495
|
-
if detected_by == "codex_explicit_injection":
|
|
496
|
-
return "codex_native_skill_injection"
|
|
497
|
-
if detected_by == "codex_implicit_script":
|
|
498
|
-
return "codex_skill_script_exec"
|
|
499
|
-
if detected_by == "codex_implicit_doc_read":
|
|
500
|
-
return "codex_skill_doc_read"
|
|
501
|
-
if detected_by == "tool_call":
|
|
502
|
-
return "codex_unsupported_skill_tool"
|
|
503
|
-
if detected_by == "skill_file_path":
|
|
504
|
-
return "skill_file_path"
|
|
505
|
-
return detected_by or "metadata"
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
def _skill_invocation_mode(agent: str, detected_by: str) -> str:
|
|
509
|
-
if agent == "codex" and detected_by == "codex_explicit_injection":
|
|
510
|
-
return "explicit"
|
|
511
|
-
if agent == "codex" and detected_by in ("codex_implicit_script", "codex_implicit_doc_read"):
|
|
512
|
-
return "implicit"
|
|
513
|
-
return "detected"
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
def _skill_event_type(detected_by: str, agent: str = "unknown") -> str:
|
|
517
|
-
if agent == "codex" and detected_by == "tool_call":
|
|
518
|
-
return "detected"
|
|
519
|
-
return "invoked" if detected_by in ("codex_explicit_injection", "codex_implicit_script", "codex_implicit_doc_read") else "detected"
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
def _skill_id_segment(name: str) -> str:
|
|
523
|
-
segment = re.sub(r"[^A-Za-z0-9_.:-]+", "-", str(name or "").strip()).strip("-")
|
|
524
|
-
return (segment or "unknown")[:96]
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
def _skill_usage(name: str, detected_by: str, skill_call_id: str = "") -> Dict[str, str]:
|
|
528
|
-
clean = str(name or "").strip()
|
|
529
|
-
return {
|
|
530
|
-
"name": clean,
|
|
531
|
-
"skill_namespace": _skill_namespace(clean),
|
|
532
|
-
"detected_by": detected_by,
|
|
533
|
-
"skill_call_id": str(skill_call_id or "").strip(),
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
_CODEX_SCRIPT_RUNNERS = {"python", "python3", "bash", "zsh", "sh", "node", "deno", "ruby", "perl", "pwsh"}
|
|
538
|
-
_CODEX_DOC_READERS = {"cat", "sed", "head", "tail", "less", "more", "bat", "awk"}
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
def _command_from_tool_input(input_obj: Any) -> str:
|
|
542
|
-
if isinstance(input_obj, dict):
|
|
543
|
-
command = input_obj.get("command")
|
|
544
|
-
if isinstance(command, str) and command.strip():
|
|
545
|
-
return command.strip()
|
|
546
|
-
argv = input_obj.get("cmd") or input_obj.get("argv")
|
|
547
|
-
if isinstance(argv, list):
|
|
548
|
-
return " ".join(str(item) for item in argv)
|
|
549
|
-
if isinstance(input_obj, str):
|
|
550
|
-
return input_obj.strip()
|
|
551
|
-
return ""
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
def _first_command_token(command: str) -> str:
|
|
555
|
-
match = re.match(r"\s*(?:[A-Za-z]:)?[^\"'\s]*?([^\\/\"'\s]+)(?:\.(?:exe|cmd))?(?=\s|$)", command, re.IGNORECASE)
|
|
556
|
-
return match.group(1).lower() if match else ""
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
def _accept_known_skill(name: str, known_skills: set) -> str:
|
|
560
|
-
clean = str(name or "").strip()
|
|
561
|
-
return clean if clean and (clean in known_skills or not known_skills) else ""
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
def _detect_codex_skill_command(command: str, known_skills: set) -> List[Dict[str, str]]:
|
|
565
|
-
found: List[Dict[str, str]] = []
|
|
566
|
-
if not command:
|
|
567
|
-
return found
|
|
568
|
-
|
|
569
|
-
first = _first_command_token(command)
|
|
570
|
-
if first in _CODEX_SCRIPT_RUNNERS:
|
|
571
|
-
script_pattern = r"[\\/](?:skills|skill)[\\/]([^\\/\"'\s]+)[\\/]scripts[\\/][^\"'\s]+\.(?:py|sh|js|ts|rb|pl|ps1)(?=$|[\"'\s])"
|
|
572
|
-
for match in re.finditer(script_pattern, command, re.IGNORECASE):
|
|
573
|
-
name = _accept_known_skill(match.group(1), known_skills)
|
|
574
|
-
if name:
|
|
575
|
-
found.append(_skill_usage(name, "codex_implicit_script"))
|
|
576
|
-
|
|
577
|
-
if first in _CODEX_DOC_READERS:
|
|
578
|
-
doc_pattern = r"[\\/](?:skills|skill)[\\/]([^\\/\"'\s]+)[\\/]SKILL\.md(?=$|[\"'\s])"
|
|
579
|
-
for match in re.finditer(doc_pattern, command, re.IGNORECASE):
|
|
580
|
-
name = _accept_known_skill(match.group(1), known_skills)
|
|
581
|
-
if name:
|
|
582
|
-
found.append(_skill_usage(name, "codex_implicit_doc_read"))
|
|
583
|
-
|
|
584
|
-
return found
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
def detect_skill_usages(tool_calls: List[Dict[str, Any]], known_skills: set) -> List[Dict[str, str]]:
|
|
588
|
-
found: List[Dict[str, str]] = []
|
|
589
|
-
for call in tool_calls or []:
|
|
590
|
-
input_obj = call.get("input") if isinstance(call.get("input"), (dict, list, str)) else {}
|
|
591
|
-
found.extend(_detect_codex_skill_command(_command_from_tool_input(input_obj), known_skills))
|
|
592
|
-
return found
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
def _detect_codex_explicit_injections(material: Dict[str, Any], known_skills: set) -> List[Dict[str, str]]:
|
|
596
|
-
found: List[Dict[str, str]] = []
|
|
597
|
-
sources = [material.get("user_text"), *(material.get("skill_detection_sources") or [])]
|
|
598
|
-
for source in sources:
|
|
599
|
-
text = extract_text(source) if not isinstance(source, str) else source
|
|
600
|
-
if not text:
|
|
601
|
-
try:
|
|
602
|
-
text = json.dumps(source, ensure_ascii=False)
|
|
603
|
-
except Exception:
|
|
604
|
-
text = str(source)
|
|
605
|
-
for match in re.finditer(r"<skill>\s*<name>\s*([^<\s]+)\s*</name>.*?</skill>", text, re.IGNORECASE | re.DOTALL):
|
|
606
|
-
name = _accept_known_skill(match.group(1), known_skills)
|
|
607
|
-
if name:
|
|
608
|
-
found.append(_skill_usage(name, "codex_explicit_injection"))
|
|
609
|
-
return found
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
def _dedupe_turn_skill_usages(usages: List[Dict[str, str]]) -> List[Dict[str, str]]:
|
|
613
|
-
out: List[Dict[str, str]] = []
|
|
614
|
-
seen_call_ids: set = set()
|
|
615
|
-
seen_detected: set = set()
|
|
616
|
-
for usage in usages or []:
|
|
617
|
-
name = str(usage.get("name") or "").strip()
|
|
618
|
-
if not name:
|
|
619
|
-
continue
|
|
620
|
-
call_id = str(usage.get("skill_call_id") or "").strip()
|
|
621
|
-
if call_id:
|
|
622
|
-
key = f"call:{call_id}"
|
|
623
|
-
if key in seen_call_ids:
|
|
624
|
-
continue
|
|
625
|
-
seen_call_ids.add(key)
|
|
626
|
-
out.append(usage)
|
|
627
|
-
continue
|
|
628
|
-
detected_by = str(usage.get("detected_by") or "")
|
|
629
|
-
if detected_by == "skill_file_path":
|
|
630
|
-
key = f"{name}:{detected_by}"
|
|
631
|
-
if key in seen_detected:
|
|
632
|
-
continue
|
|
633
|
-
seen_detected.add(key)
|
|
634
|
-
out.append(usage)
|
|
635
|
-
return out
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
def detect_turn_skill_usages(material: Dict[str, Any], known_skills: set) -> List[Dict[str, str]]:
|
|
639
|
-
found = list(_detect_codex_explicit_injections(material, known_skills))
|
|
640
|
-
found.extend(detect_skill_usages(material.get("tool_calls") or [], known_skills))
|
|
641
|
-
return _dedupe_turn_skill_usages(found)
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
def build_skill_use_events(interaction_id: str, skill_usages: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
645
|
-
events: List[Dict[str, Any]] = []
|
|
646
|
-
deduped: List[Dict[str, str]] = []
|
|
647
|
-
seen_call_ids: set = set()
|
|
648
|
-
agent = _skill_agent_from_interaction_id(interaction_id)
|
|
649
|
-
for skill in skill_usages or []:
|
|
650
|
-
call_id = str(skill.get("skill_call_id") or "").strip()
|
|
651
|
-
if call_id:
|
|
652
|
-
dedupe_key = f"call:{call_id}"
|
|
653
|
-
if dedupe_key in seen_call_ids:
|
|
654
|
-
continue
|
|
655
|
-
seen_call_ids.add(dedupe_key)
|
|
656
|
-
deduped.append(skill)
|
|
657
|
-
total = len(deduped)
|
|
658
|
-
for index, skill in enumerate(deduped, start=1):
|
|
659
|
-
name = str(skill.get("name") or "").strip()
|
|
660
|
-
if not name:
|
|
661
|
-
continue
|
|
662
|
-
detected_by = str(skill.get("detected_by") or "metadata")
|
|
663
|
-
call_id = str(skill.get("skill_call_id") or "").strip()
|
|
664
|
-
invocation_mode = _skill_invocation_mode(agent, detected_by)
|
|
665
|
-
events.append({
|
|
666
|
-
"skill_use_id": f"{interaction_id}:skill:{index}:{_skill_id_segment(name)}",
|
|
667
|
-
"skill_use_index": index,
|
|
668
|
-
"skill_use_count_in_interaction": total,
|
|
669
|
-
"skill_event_type": _skill_event_type(detected_by, agent),
|
|
670
|
-
"skill_trigger": invocation_mode,
|
|
671
|
-
"skill_invocation_mode": invocation_mode,
|
|
672
|
-
"skill_agent_path": _skill_agent_path(agent, detected_by),
|
|
673
|
-
"skill_name": name,
|
|
674
|
-
"skill_use_count": 1,
|
|
675
|
-
"skill_namespace": skill.get("skill_namespace") or _skill_namespace(name),
|
|
676
|
-
"detected_by": detected_by,
|
|
677
|
-
**({"skill_call_id": call_id} if call_id else {}),
|
|
678
|
-
})
|
|
679
|
-
return events
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
def summarize_skill_usages(skill_usages: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
683
|
-
summary: Dict[str, Dict[str, Any]] = {}
|
|
684
|
-
for item in skill_usages or []:
|
|
685
|
-
name = item.get("name")
|
|
686
|
-
if not name:
|
|
687
|
-
continue
|
|
688
|
-
entry = summary.setdefault(name, {"name": name, "count": 0, "detected_by": item.get("detected_by")})
|
|
689
|
-
entry["count"] += 1
|
|
690
|
-
detected_by = str(item.get("detected_by") or "metadata")
|
|
691
|
-
entry.setdefault("skill_invocation_mode", _skill_invocation_mode("codex", detected_by))
|
|
692
|
-
entry.setdefault("skill_agent_path", _skill_agent_path("codex", detected_by))
|
|
693
|
-
return list(summary.values())
|
|
337
|
+
return kept, {
|
|
338
|
+
"truncated": True,
|
|
339
|
+
"orig_len": orig_len,
|
|
340
|
+
"kept_len": len(kept),
|
|
341
|
+
"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def build_interaction_id(source: str, session_id: str, turn_number: int) -> str:
|
|
346
|
+
return f"{source or 'unknown'}:{session_id or 'unknown'}:{int(turn_number or 0)}"
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _num_or_none(value: Any) -> Optional[int]:
|
|
350
|
+
if isinstance(value, bool):
|
|
351
|
+
return None
|
|
352
|
+
if isinstance(value, int) and value >= 0:
|
|
353
|
+
return value
|
|
354
|
+
if isinstance(value, float) and value >= 0:
|
|
355
|
+
return int(value)
|
|
356
|
+
if isinstance(value, str):
|
|
357
|
+
try:
|
|
358
|
+
n = int(value)
|
|
359
|
+
return n if n >= 0 else None
|
|
360
|
+
except Exception:
|
|
361
|
+
return None
|
|
362
|
+
return None
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _first_num(raw: Dict[str, Any], *keys: str) -> Optional[int]:
|
|
366
|
+
for key in keys:
|
|
367
|
+
if key in raw:
|
|
368
|
+
value = _num_or_none(raw.get(key))
|
|
369
|
+
if value is not None:
|
|
370
|
+
return value
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def normalize_token_metrics(raw: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
375
|
+
if not isinstance(raw, dict) or not raw:
|
|
376
|
+
return {
|
|
377
|
+
"token_metrics_available": False,
|
|
378
|
+
"input_tokens": None,
|
|
379
|
+
"output_tokens": None,
|
|
380
|
+
"total_tokens": None,
|
|
381
|
+
"cache_read_tokens": None,
|
|
382
|
+
"reasoning_tokens": None,
|
|
383
|
+
}
|
|
384
|
+
input_tokens = _first_num(raw, "input", "input_tokens", "inputTokens")
|
|
385
|
+
output_tokens = _first_num(raw, "output", "output_tokens", "outputTokens")
|
|
386
|
+
total_tokens = _first_num(raw, "total", "total_tokens", "totalTokens")
|
|
387
|
+
if total_tokens is None and input_tokens is not None and output_tokens is not None:
|
|
388
|
+
total_tokens = input_tokens + output_tokens
|
|
389
|
+
cache_read_tokens = _first_num(raw, "cache_read_tokens", "cachedInputTokens", "cacheRead")
|
|
390
|
+
reasoning_tokens = _first_num(raw, "reasoning_tokens", "reasoningTokens", "reasoning")
|
|
391
|
+
available = any(v is not None for v in [input_tokens, output_tokens, total_tokens, cache_read_tokens, reasoning_tokens])
|
|
392
|
+
return {
|
|
393
|
+
"token_metrics_available": available,
|
|
394
|
+
"input_tokens": input_tokens if available else None,
|
|
395
|
+
"output_tokens": output_tokens if available else None,
|
|
396
|
+
"total_tokens": total_tokens if available else None,
|
|
397
|
+
"cache_read_tokens": cache_read_tokens if available else None,
|
|
398
|
+
"reasoning_tokens": reasoning_tokens if available else None,
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _ratio(numerator: Optional[int], denominator: Optional[int]) -> Optional[float]:
|
|
403
|
+
if numerator is None or denominator in (None, 0):
|
|
404
|
+
return None
|
|
405
|
+
return numerator / denominator
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def build_interaction_metadata(
|
|
409
|
+
source: str,
|
|
410
|
+
user_id: Optional[str],
|
|
411
|
+
session_id: str,
|
|
412
|
+
turn_number: int,
|
|
413
|
+
token_metrics: Optional[Dict[str, Any]],
|
|
414
|
+
tool_call_count: int,
|
|
415
|
+
tool_result_count: int,
|
|
416
|
+
skill_use_count: int,
|
|
417
|
+
model: Optional[str],
|
|
418
|
+
user_message_count: int = 1,
|
|
419
|
+
assistant_message_count: int = 1,
|
|
420
|
+
skill_use_events: Optional[List[Dict[str, Any]]] = None,
|
|
421
|
+
) -> Dict[str, Any]:
|
|
422
|
+
tokens = normalize_token_metrics(token_metrics)
|
|
423
|
+
interaction_id = build_interaction_id(source, session_id, turn_number)
|
|
424
|
+
events = list(skill_use_events or [])
|
|
425
|
+
skill_names_all = [str(event.get("skill_name") or "") for event in events if event.get("skill_name")]
|
|
426
|
+
unique_skill_names = list(dict.fromkeys(skill_names_all))
|
|
427
|
+
skill_invocation_modes = [str(event.get("skill_invocation_mode") or "") for event in events if event.get("skill_invocation_mode")]
|
|
428
|
+
skill_agent_paths = [str(event.get("skill_agent_path") or "") for event in events if event.get("skill_agent_path")]
|
|
429
|
+
effective_skill_count = len(events) if events else int(skill_use_count or 0)
|
|
430
|
+
return {
|
|
431
|
+
"source": source,
|
|
432
|
+
"agent": source,
|
|
433
|
+
"user_id": user_id or "",
|
|
434
|
+
"session_id": session_id,
|
|
435
|
+
"interaction_id": interaction_id,
|
|
436
|
+
"metrics_schema_version": METRICS_SCHEMA_VERSION,
|
|
437
|
+
"interaction_count": 1,
|
|
438
|
+
"user_message_count": user_message_count,
|
|
439
|
+
"assistant_message_count": assistant_message_count,
|
|
440
|
+
"tool_call_count": int(tool_call_count or 0),
|
|
441
|
+
"tool_result_count": int(tool_result_count or 0),
|
|
442
|
+
"skill_use_count": effective_skill_count,
|
|
443
|
+
"unique_skill_count": len(unique_skill_names),
|
|
444
|
+
"repeated_skill_count": max(0, effective_skill_count - len(unique_skill_names)),
|
|
445
|
+
**tokens,
|
|
446
|
+
"model": model,
|
|
447
|
+
"turn_number": int(turn_number or 0),
|
|
448
|
+
"efficiency": {
|
|
449
|
+
"tokens_per_interaction": tokens.get("total_tokens"),
|
|
450
|
+
"tool_calls_per_interaction": int(tool_call_count or 0),
|
|
451
|
+
"skills_per_interaction": effective_skill_count,
|
|
452
|
+
"output_input_token_ratio": _ratio(tokens.get("output_tokens"), tokens.get("input_tokens")),
|
|
453
|
+
"tokens_per_tool_call": _ratio(tokens.get("total_tokens"), int(tool_call_count or 0)),
|
|
454
|
+
},
|
|
455
|
+
**({
|
|
456
|
+
"skill_names": unique_skill_names,
|
|
457
|
+
"skill_names_all": skill_names_all,
|
|
458
|
+
"skill_invocation_modes": skill_invocation_modes,
|
|
459
|
+
"skill_agent_paths": skill_agent_paths,
|
|
460
|
+
} if events else {}),
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def discover_known_skills(extra_roots: Optional[List[Path]] = None) -> set:
|
|
465
|
+
roots = [
|
|
466
|
+
CODEX_DIR / "skills",
|
|
467
|
+
CODEX_DIR / "plugins" / "cache",
|
|
468
|
+
Path.home() / ".claude" / "skills",
|
|
469
|
+
Path.home() / ".config" / "opencode" / "skill",
|
|
470
|
+
]
|
|
471
|
+
if extra_roots:
|
|
472
|
+
roots.extend(extra_roots)
|
|
473
|
+
names = set()
|
|
474
|
+
for root in roots:
|
|
475
|
+
try:
|
|
476
|
+
if not root.exists():
|
|
477
|
+
continue
|
|
478
|
+
for skill_file in root.rglob("SKILL.md"):
|
|
479
|
+
names.add(skill_file.parent.name)
|
|
480
|
+
except Exception:
|
|
481
|
+
continue
|
|
482
|
+
return names
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _skill_namespace(name: str) -> str:
|
|
486
|
+
return name.split(":", 1)[0] if ":" in name else ""
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _skill_agent_from_interaction_id(interaction_id: str) -> str:
|
|
490
|
+
return str(interaction_id or "unknown").split(":", 1)[0] or "unknown"
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _skill_agent_path(agent: str, detected_by: str) -> str:
|
|
494
|
+
if agent == "codex":
|
|
495
|
+
if detected_by == "codex_explicit_injection":
|
|
496
|
+
return "codex_native_skill_injection"
|
|
497
|
+
if detected_by == "codex_implicit_script":
|
|
498
|
+
return "codex_skill_script_exec"
|
|
499
|
+
if detected_by == "codex_implicit_doc_read":
|
|
500
|
+
return "codex_skill_doc_read"
|
|
501
|
+
if detected_by == "tool_call":
|
|
502
|
+
return "codex_unsupported_skill_tool"
|
|
503
|
+
if detected_by == "skill_file_path":
|
|
504
|
+
return "skill_file_path"
|
|
505
|
+
return detected_by or "metadata"
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _skill_invocation_mode(agent: str, detected_by: str) -> str:
|
|
509
|
+
if agent == "codex" and detected_by == "codex_explicit_injection":
|
|
510
|
+
return "explicit"
|
|
511
|
+
if agent == "codex" and detected_by in ("codex_implicit_script", "codex_implicit_doc_read"):
|
|
512
|
+
return "implicit"
|
|
513
|
+
return "detected"
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _skill_event_type(detected_by: str, agent: str = "unknown") -> str:
|
|
517
|
+
if agent == "codex" and detected_by == "tool_call":
|
|
518
|
+
return "detected"
|
|
519
|
+
return "invoked" if detected_by in ("codex_explicit_injection", "codex_implicit_script", "codex_implicit_doc_read") else "detected"
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _skill_id_segment(name: str) -> str:
|
|
523
|
+
segment = re.sub(r"[^A-Za-z0-9_.:-]+", "-", str(name or "").strip()).strip("-")
|
|
524
|
+
return (segment or "unknown")[:96]
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _skill_usage(name: str, detected_by: str, skill_call_id: str = "") -> Dict[str, str]:
|
|
528
|
+
clean = str(name or "").strip()
|
|
529
|
+
return {
|
|
530
|
+
"name": clean,
|
|
531
|
+
"skill_namespace": _skill_namespace(clean),
|
|
532
|
+
"detected_by": detected_by,
|
|
533
|
+
"skill_call_id": str(skill_call_id or "").strip(),
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
_CODEX_SCRIPT_RUNNERS = {"python", "python3", "bash", "zsh", "sh", "node", "deno", "ruby", "perl", "pwsh"}
|
|
538
|
+
_CODEX_DOC_READERS = {"cat", "sed", "head", "tail", "less", "more", "bat", "awk"}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _command_from_tool_input(input_obj: Any) -> str:
|
|
542
|
+
if isinstance(input_obj, dict):
|
|
543
|
+
command = input_obj.get("command")
|
|
544
|
+
if isinstance(command, str) and command.strip():
|
|
545
|
+
return command.strip()
|
|
546
|
+
argv = input_obj.get("cmd") or input_obj.get("argv")
|
|
547
|
+
if isinstance(argv, list):
|
|
548
|
+
return " ".join(str(item) for item in argv)
|
|
549
|
+
if isinstance(input_obj, str):
|
|
550
|
+
return input_obj.strip()
|
|
551
|
+
return ""
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _first_command_token(command: str) -> str:
|
|
555
|
+
match = re.match(r"\s*(?:[A-Za-z]:)?[^\"'\s]*?([^\\/\"'\s]+)(?:\.(?:exe|cmd))?(?=\s|$)", command, re.IGNORECASE)
|
|
556
|
+
return match.group(1).lower() if match else ""
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _accept_known_skill(name: str, known_skills: set) -> str:
|
|
560
|
+
clean = str(name or "").strip()
|
|
561
|
+
return clean if clean and (clean in known_skills or not known_skills) else ""
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _detect_codex_skill_command(command: str, known_skills: set) -> List[Dict[str, str]]:
|
|
565
|
+
found: List[Dict[str, str]] = []
|
|
566
|
+
if not command:
|
|
567
|
+
return found
|
|
568
|
+
|
|
569
|
+
first = _first_command_token(command)
|
|
570
|
+
if first in _CODEX_SCRIPT_RUNNERS:
|
|
571
|
+
script_pattern = r"[\\/](?:skills|skill)[\\/]([^\\/\"'\s]+)[\\/]scripts[\\/][^\"'\s]+\.(?:py|sh|js|ts|rb|pl|ps1)(?=$|[\"'\s])"
|
|
572
|
+
for match in re.finditer(script_pattern, command, re.IGNORECASE):
|
|
573
|
+
name = _accept_known_skill(match.group(1), known_skills)
|
|
574
|
+
if name:
|
|
575
|
+
found.append(_skill_usage(name, "codex_implicit_script"))
|
|
576
|
+
|
|
577
|
+
if first in _CODEX_DOC_READERS:
|
|
578
|
+
doc_pattern = r"[\\/](?:skills|skill)[\\/]([^\\/\"'\s]+)[\\/]SKILL\.md(?=$|[\"'\s])"
|
|
579
|
+
for match in re.finditer(doc_pattern, command, re.IGNORECASE):
|
|
580
|
+
name = _accept_known_skill(match.group(1), known_skills)
|
|
581
|
+
if name:
|
|
582
|
+
found.append(_skill_usage(name, "codex_implicit_doc_read"))
|
|
583
|
+
|
|
584
|
+
return found
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def detect_skill_usages(tool_calls: List[Dict[str, Any]], known_skills: set) -> List[Dict[str, str]]:
|
|
588
|
+
found: List[Dict[str, str]] = []
|
|
589
|
+
for call in tool_calls or []:
|
|
590
|
+
input_obj = call.get("input") if isinstance(call.get("input"), (dict, list, str)) else {}
|
|
591
|
+
found.extend(_detect_codex_skill_command(_command_from_tool_input(input_obj), known_skills))
|
|
592
|
+
return found
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _detect_codex_explicit_injections(material: Dict[str, Any], known_skills: set) -> List[Dict[str, str]]:
|
|
596
|
+
found: List[Dict[str, str]] = []
|
|
597
|
+
sources = [material.get("user_text"), *(material.get("skill_detection_sources") or [])]
|
|
598
|
+
for source in sources:
|
|
599
|
+
text = extract_text(source) if not isinstance(source, str) else source
|
|
600
|
+
if not text:
|
|
601
|
+
try:
|
|
602
|
+
text = json.dumps(source, ensure_ascii=False)
|
|
603
|
+
except Exception:
|
|
604
|
+
text = str(source)
|
|
605
|
+
for match in re.finditer(r"<skill>\s*<name>\s*([^<\s]+)\s*</name>.*?</skill>", text, re.IGNORECASE | re.DOTALL):
|
|
606
|
+
name = _accept_known_skill(match.group(1), known_skills)
|
|
607
|
+
if name:
|
|
608
|
+
found.append(_skill_usage(name, "codex_explicit_injection"))
|
|
609
|
+
return found
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _dedupe_turn_skill_usages(usages: List[Dict[str, str]]) -> List[Dict[str, str]]:
|
|
613
|
+
out: List[Dict[str, str]] = []
|
|
614
|
+
seen_call_ids: set = set()
|
|
615
|
+
seen_detected: set = set()
|
|
616
|
+
for usage in usages or []:
|
|
617
|
+
name = str(usage.get("name") or "").strip()
|
|
618
|
+
if not name:
|
|
619
|
+
continue
|
|
620
|
+
call_id = str(usage.get("skill_call_id") or "").strip()
|
|
621
|
+
if call_id:
|
|
622
|
+
key = f"call:{call_id}"
|
|
623
|
+
if key in seen_call_ids:
|
|
624
|
+
continue
|
|
625
|
+
seen_call_ids.add(key)
|
|
626
|
+
out.append(usage)
|
|
627
|
+
continue
|
|
628
|
+
detected_by = str(usage.get("detected_by") or "")
|
|
629
|
+
if detected_by == "skill_file_path":
|
|
630
|
+
key = f"{name}:{detected_by}"
|
|
631
|
+
if key in seen_detected:
|
|
632
|
+
continue
|
|
633
|
+
seen_detected.add(key)
|
|
634
|
+
out.append(usage)
|
|
635
|
+
return out
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def detect_turn_skill_usages(material: Dict[str, Any], known_skills: set) -> List[Dict[str, str]]:
|
|
639
|
+
found = list(_detect_codex_explicit_injections(material, known_skills))
|
|
640
|
+
found.extend(detect_skill_usages(material.get("tool_calls") or [], known_skills))
|
|
641
|
+
return _dedupe_turn_skill_usages(found)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def build_skill_use_events(interaction_id: str, skill_usages: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
645
|
+
events: List[Dict[str, Any]] = []
|
|
646
|
+
deduped: List[Dict[str, str]] = []
|
|
647
|
+
seen_call_ids: set = set()
|
|
648
|
+
agent = _skill_agent_from_interaction_id(interaction_id)
|
|
649
|
+
for skill in skill_usages or []:
|
|
650
|
+
call_id = str(skill.get("skill_call_id") or "").strip()
|
|
651
|
+
if call_id:
|
|
652
|
+
dedupe_key = f"call:{call_id}"
|
|
653
|
+
if dedupe_key in seen_call_ids:
|
|
654
|
+
continue
|
|
655
|
+
seen_call_ids.add(dedupe_key)
|
|
656
|
+
deduped.append(skill)
|
|
657
|
+
total = len(deduped)
|
|
658
|
+
for index, skill in enumerate(deduped, start=1):
|
|
659
|
+
name = str(skill.get("name") or "").strip()
|
|
660
|
+
if not name:
|
|
661
|
+
continue
|
|
662
|
+
detected_by = str(skill.get("detected_by") or "metadata")
|
|
663
|
+
call_id = str(skill.get("skill_call_id") or "").strip()
|
|
664
|
+
invocation_mode = _skill_invocation_mode(agent, detected_by)
|
|
665
|
+
events.append({
|
|
666
|
+
"skill_use_id": f"{interaction_id}:skill:{index}:{_skill_id_segment(name)}",
|
|
667
|
+
"skill_use_index": index,
|
|
668
|
+
"skill_use_count_in_interaction": total,
|
|
669
|
+
"skill_event_type": _skill_event_type(detected_by, agent),
|
|
670
|
+
"skill_trigger": invocation_mode,
|
|
671
|
+
"skill_invocation_mode": invocation_mode,
|
|
672
|
+
"skill_agent_path": _skill_agent_path(agent, detected_by),
|
|
673
|
+
"skill_name": name,
|
|
674
|
+
"skill_use_count": 1,
|
|
675
|
+
"skill_namespace": skill.get("skill_namespace") or _skill_namespace(name),
|
|
676
|
+
"detected_by": detected_by,
|
|
677
|
+
**({"skill_call_id": call_id} if call_id else {}),
|
|
678
|
+
})
|
|
679
|
+
return events
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def summarize_skill_usages(skill_usages: List[Dict[str, str]]) -> List[Dict[str, Any]]:
|
|
683
|
+
summary: Dict[str, Dict[str, Any]] = {}
|
|
684
|
+
for item in skill_usages or []:
|
|
685
|
+
name = item.get("name")
|
|
686
|
+
if not name:
|
|
687
|
+
continue
|
|
688
|
+
entry = summary.setdefault(name, {"name": name, "count": 0, "detected_by": item.get("detected_by")})
|
|
689
|
+
entry["count"] += 1
|
|
690
|
+
detected_by = str(item.get("detected_by") or "metadata")
|
|
691
|
+
entry.setdefault("skill_invocation_mode", _skill_invocation_mode("codex", detected_by))
|
|
692
|
+
entry.setdefault("skill_agent_path", _skill_agent_path("codex", detected_by))
|
|
693
|
+
return list(summary.values())
|
|
694
694
|
|
|
695
695
|
|
|
696
696
|
def get_payload(row: Dict[str, Any]) -> Dict[str, Any]:
|
|
@@ -743,16 +743,16 @@ def usage_details_from_codex(usage: Dict[str, Any]) -> Dict[str, int]:
|
|
|
743
743
|
|
|
744
744
|
|
|
745
745
|
def collect_turn_material(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
746
|
-
user_texts: List[str] = []
|
|
747
|
-
assistant_texts: List[str] = []
|
|
748
|
-
tool_calls: List[Dict[str, Any]] = []
|
|
749
|
-
tool_results: List[Dict[str, Any]] = []
|
|
750
|
-
skill_detection_sources: List[Any] = []
|
|
751
|
-
|
|
752
|
-
for row in rows:
|
|
753
|
-
row_type = row.get("type")
|
|
754
|
-
payload = get_payload(row)
|
|
755
|
-
skill_detection_sources.append(payload or row)
|
|
746
|
+
user_texts: List[str] = []
|
|
747
|
+
assistant_texts: List[str] = []
|
|
748
|
+
tool_calls: List[Dict[str, Any]] = []
|
|
749
|
+
tool_results: List[Dict[str, Any]] = []
|
|
750
|
+
skill_detection_sources: List[Any] = []
|
|
751
|
+
|
|
752
|
+
for row in rows:
|
|
753
|
+
row_type = row.get("type")
|
|
754
|
+
payload = get_payload(row)
|
|
755
|
+
skill_detection_sources.append(payload or row)
|
|
756
756
|
|
|
757
757
|
if row_type == "response_item":
|
|
758
758
|
item_type = payload.get("type")
|
|
@@ -794,11 +794,11 @@ def collect_turn_material(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
|
794
794
|
|
|
795
795
|
return {
|
|
796
796
|
"user_text": "\n\n".join(user_texts[-3:]),
|
|
797
|
-
"assistant_text": "\n\n".join(assistant_texts),
|
|
798
|
-
"tool_calls": tool_calls,
|
|
799
|
-
"tool_results": tool_results,
|
|
800
|
-
"skill_detection_sources": skill_detection_sources,
|
|
801
|
-
}
|
|
797
|
+
"assistant_text": "\n\n".join(assistant_texts),
|
|
798
|
+
"tool_calls": tool_calls,
|
|
799
|
+
"tool_results": tool_results,
|
|
800
|
+
"skill_detection_sources": skill_detection_sources,
|
|
801
|
+
}
|
|
802
802
|
|
|
803
803
|
|
|
804
804
|
def emit_codex_turn(
|
|
@@ -813,113 +813,113 @@ def emit_codex_turn(
|
|
|
813
813
|
) -> None:
|
|
814
814
|
user_text, user_meta = truncate(material.get("user_text") or "")
|
|
815
815
|
assistant_text, assistant_meta = truncate(material.get("assistant_text") or "")
|
|
816
|
-
usage_details = usage_details_from_codex(usage)
|
|
817
|
-
model = first_string(meta.get("model"), meta.get("model_provider")) or "codex"
|
|
818
|
-
tool_calls = material.get("tool_calls") or []
|
|
819
|
-
tool_results = material.get("tool_results") or []
|
|
820
|
-
skill_usages = detect_turn_skill_usages(material, discover_known_skills())
|
|
821
|
-
interaction_id = build_interaction_id("codex", session_id, turn_num)
|
|
822
|
-
skill_use_events = build_skill_use_events(interaction_id, skill_usages)
|
|
823
|
-
interaction_meta = build_interaction_metadata(
|
|
824
|
-
"codex",
|
|
825
|
-
user_id,
|
|
826
|
-
session_id,
|
|
827
|
-
turn_num,
|
|
828
|
-
usage_details,
|
|
829
|
-
len(tool_calls),
|
|
830
|
-
len(tool_results),
|
|
831
|
-
len(skill_use_events),
|
|
832
|
-
model,
|
|
833
|
-
user_message_count=1 if material.get("user_text") else 0,
|
|
834
|
-
assistant_message_count=1 if material.get("assistant_text") else 0,
|
|
835
|
-
skill_use_events=skill_use_events,
|
|
836
|
-
)
|
|
837
|
-
skill_summary = summarize_skill_usages(skill_usages)
|
|
838
|
-
|
|
839
|
-
with propagate_attributes(
|
|
840
|
-
user_id=user_id,
|
|
816
|
+
usage_details = usage_details_from_codex(usage)
|
|
817
|
+
model = first_string(meta.get("model"), meta.get("model_provider")) or "codex"
|
|
818
|
+
tool_calls = material.get("tool_calls") or []
|
|
819
|
+
tool_results = material.get("tool_results") or []
|
|
820
|
+
skill_usages = detect_turn_skill_usages(material, discover_known_skills())
|
|
821
|
+
interaction_id = build_interaction_id("codex", session_id, turn_num)
|
|
822
|
+
skill_use_events = build_skill_use_events(interaction_id, skill_usages)
|
|
823
|
+
interaction_meta = build_interaction_metadata(
|
|
824
|
+
"codex",
|
|
825
|
+
user_id,
|
|
826
|
+
session_id,
|
|
827
|
+
turn_num,
|
|
828
|
+
usage_details,
|
|
829
|
+
len(tool_calls),
|
|
830
|
+
len(tool_results),
|
|
831
|
+
len(skill_use_events),
|
|
832
|
+
model,
|
|
833
|
+
user_message_count=1 if material.get("user_text") else 0,
|
|
834
|
+
assistant_message_count=1 if material.get("assistant_text") else 0,
|
|
835
|
+
skill_use_events=skill_use_events,
|
|
836
|
+
)
|
|
837
|
+
skill_summary = summarize_skill_usages(skill_usages)
|
|
838
|
+
|
|
839
|
+
with propagate_attributes(
|
|
840
|
+
user_id=user_id,
|
|
841
841
|
session_id=session_id,
|
|
842
|
-
trace_name="Agent Turn",
|
|
843
|
-
tags=[AGENT_NAME],
|
|
842
|
+
trace_name="Agent Turn",
|
|
843
|
+
tags=[AGENT_NAME],
|
|
844
844
|
):
|
|
845
|
-
with langfuse.start_as_current_observation(
|
|
846
|
-
name="Agent Turn",
|
|
847
|
-
input={"role": "user", "content": user_text},
|
|
848
|
-
output={"role": "assistant", "content": assistant_text},
|
|
849
|
-
metadata={
|
|
850
|
-
**interaction_meta,
|
|
851
|
-
"source": AGENT_NAME,
|
|
852
|
-
"agent": AGENT_NAME,
|
|
853
|
-
"session_id": session_id,
|
|
854
|
-
"turn_number": turn_num,
|
|
855
|
-
"session_path": str(session_path),
|
|
845
|
+
with langfuse.start_as_current_observation(
|
|
846
|
+
name="Agent Turn",
|
|
847
|
+
input={"role": "user", "content": user_text},
|
|
848
|
+
output={"role": "assistant", "content": assistant_text},
|
|
849
|
+
metadata={
|
|
850
|
+
**interaction_meta,
|
|
851
|
+
"source": AGENT_NAME,
|
|
852
|
+
"agent": AGENT_NAME,
|
|
853
|
+
"session_id": session_id,
|
|
854
|
+
"turn_number": turn_num,
|
|
855
|
+
"session_path": str(session_path),
|
|
856
856
|
"cwd": meta.get("cwd"),
|
|
857
857
|
"originator": meta.get("originator"),
|
|
858
|
-
"cli_version": meta.get("cli_version"),
|
|
859
|
-
"user_text": user_meta,
|
|
860
|
-
"usage": usage,
|
|
861
|
-
"skills": skill_summary,
|
|
862
|
-
},
|
|
863
|
-
) as trace_span:
|
|
864
|
-
with langfuse.start_as_current_observation(
|
|
865
|
-
name="Agent Response",
|
|
866
|
-
as_type="generation",
|
|
858
|
+
"cli_version": meta.get("cli_version"),
|
|
859
|
+
"user_text": user_meta,
|
|
860
|
+
"usage": usage,
|
|
861
|
+
"skills": skill_summary,
|
|
862
|
+
},
|
|
863
|
+
) as trace_span:
|
|
864
|
+
with langfuse.start_as_current_observation(
|
|
865
|
+
name="Agent Response",
|
|
866
|
+
as_type="generation",
|
|
867
867
|
model=model,
|
|
868
868
|
input={"role": "user", "content": user_text},
|
|
869
|
-
output={"role": "assistant", "content": assistant_text},
|
|
870
|
-
usage_details=usage_details or None,
|
|
871
|
-
metadata={
|
|
872
|
-
"assistant_text": assistant_meta,
|
|
873
|
-
"source": AGENT_NAME,
|
|
874
|
-
"agent": AGENT_NAME,
|
|
875
|
-
"user_id": user_id or "",
|
|
876
|
-
"session_id": session_id,
|
|
877
|
-
"interaction_id": interaction_meta["interaction_id"],
|
|
878
|
-
"turn_number": turn_num,
|
|
879
|
-
},
|
|
880
|
-
):
|
|
881
|
-
pass
|
|
882
|
-
|
|
883
|
-
for call in tool_calls:
|
|
884
|
-
tool_input, input_meta = truncate(call.get("input"))
|
|
885
|
-
with langfuse.start_as_current_observation(
|
|
886
|
-
name="Tool Call",
|
|
869
|
+
output={"role": "assistant", "content": assistant_text},
|
|
870
|
+
usage_details=usage_details or None,
|
|
871
|
+
metadata={
|
|
872
|
+
"assistant_text": assistant_meta,
|
|
873
|
+
"source": AGENT_NAME,
|
|
874
|
+
"agent": AGENT_NAME,
|
|
875
|
+
"user_id": user_id or "",
|
|
876
|
+
"session_id": session_id,
|
|
877
|
+
"interaction_id": interaction_meta["interaction_id"],
|
|
878
|
+
"turn_number": turn_num,
|
|
879
|
+
},
|
|
880
|
+
):
|
|
881
|
+
pass
|
|
882
|
+
|
|
883
|
+
for call in tool_calls:
|
|
884
|
+
tool_input, input_meta = truncate(call.get("input"))
|
|
885
|
+
with langfuse.start_as_current_observation(
|
|
886
|
+
name="Tool Call",
|
|
887
|
+
as_type="tool",
|
|
888
|
+
input=tool_input,
|
|
889
|
+
metadata={
|
|
890
|
+
"source": AGENT_NAME,
|
|
891
|
+
"agent": AGENT_NAME,
|
|
892
|
+
"user_id": user_id or "",
|
|
893
|
+
"session_id": session_id,
|
|
894
|
+
"interaction_id": interaction_meta["interaction_id"],
|
|
895
|
+
"tool_id": call.get("id"),
|
|
896
|
+
"tool_name": call.get("name"),
|
|
897
|
+
"turn_number": turn_num,
|
|
898
|
+
"input_meta": input_meta,
|
|
899
|
+
"metrics_schema_version": METRICS_SCHEMA_VERSION,
|
|
900
|
+
},
|
|
901
|
+
):
|
|
902
|
+
pass
|
|
903
|
+
|
|
904
|
+
for result in tool_results:
|
|
905
|
+
output, output_meta = truncate(result.get("output"))
|
|
906
|
+
with langfuse.start_as_current_observation(
|
|
907
|
+
name="Tool Result",
|
|
887
908
|
as_type="tool",
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
"
|
|
891
|
-
"
|
|
892
|
-
"
|
|
893
|
-
"
|
|
894
|
-
"
|
|
895
|
-
"
|
|
896
|
-
"
|
|
897
|
-
"
|
|
898
|
-
"
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
pass
|
|
903
|
-
|
|
904
|
-
for result in tool_results:
|
|
905
|
-
output, output_meta = truncate(result.get("output"))
|
|
906
|
-
with langfuse.start_as_current_observation(
|
|
907
|
-
name="Tool Result",
|
|
908
|
-
as_type="tool",
|
|
909
|
-
metadata={
|
|
910
|
-
"source": AGENT_NAME,
|
|
911
|
-
"agent": AGENT_NAME,
|
|
912
|
-
"user_id": user_id or "",
|
|
913
|
-
"session_id": session_id,
|
|
914
|
-
"interaction_id": interaction_meta["interaction_id"],
|
|
915
|
-
"tool_id": result.get("id"),
|
|
916
|
-
"tool_name": result.get("name"),
|
|
917
|
-
"turn_number": turn_num,
|
|
918
|
-
"output_meta": output_meta,
|
|
919
|
-
"metrics_schema_version": METRICS_SCHEMA_VERSION,
|
|
920
|
-
},
|
|
921
|
-
) as tool_obs:
|
|
922
|
-
tool_obs.update(output=output)
|
|
909
|
+
metadata={
|
|
910
|
+
"source": AGENT_NAME,
|
|
911
|
+
"agent": AGENT_NAME,
|
|
912
|
+
"user_id": user_id or "",
|
|
913
|
+
"session_id": session_id,
|
|
914
|
+
"interaction_id": interaction_meta["interaction_id"],
|
|
915
|
+
"tool_id": result.get("id"),
|
|
916
|
+
"tool_name": result.get("name"),
|
|
917
|
+
"turn_number": turn_num,
|
|
918
|
+
"output_meta": output_meta,
|
|
919
|
+
"metrics_schema_version": METRICS_SCHEMA_VERSION,
|
|
920
|
+
},
|
|
921
|
+
) as tool_obs:
|
|
922
|
+
tool_obs.update(output=output)
|
|
923
923
|
|
|
924
924
|
trace_span.update(output={"role": "assistant", "content": assistant_text})
|
|
925
925
|
|