@agenthippo-ai/agenthippo 1.110.11 → 1.110.20

package/out/scripts/analytics/index-traces.py

@@ -1,27 +1,6 @@
-#!/usr/bin/env python3
-# Copyright (c) AgentHippo.ai. All rights reserved.
-"""
-Incremental JSONL → DuckDB indexing worker.
-
-Converts OpenTelemetry JSONL trace files to indexed DuckDB database for fast analytics queries.
-
-Usage:
-  python3 index-traces.py <traces_dir>
-  python3 index-traces.py <traces_dir> <output_db_path>
-
-Environment variables:
-  AGENTIDE_TRACES_DIR - Path to traces directory (can also be passed as arg)
-  AGENTIDE_INDEX_DB_PATH - Path to output .duckdb file
-
-Exit codes:
-  0 = success (indexing completed)
-  1 = error (indexing failed)
-  2 = already indexed (no changes needed)
-  3 = indexing already in progress
-"""
-
+# Copyright (c) 2026 AgentHippo.ai. All rights reserved.
+# Proprietary and confidential. See LICENSE.
 from __future__ import annotations
-
 import ast
 import hashlib
 import json
@@ -31,66 +10,39 @@ import sys
 from datetime import datetime
 from pathlib import Path
 from typing import Any
-
-# Setup logging
-logging.basicConfig(
-    level=logging.DEBUG,
-    format='%(levelname)s: %(message)s',
-    stream=sys.stderr,
-)
+logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s', stream=sys.stderr)
 log = logging.getLogger(__name__)
 
-
-def get_traces_dir(arg_traces_dir: str | None = None) -> Path:
-    """Get traces directory from arg, env var, or default."""
+def get_traces_dir(arg_traces_dir: str | None=None) -> Path:
     if arg_traces_dir:
         return Path(arg_traces_dir)
-
     env_traces_dir = os.environ.get('AGENTIDE_TRACES_DIR', '').strip()
     if env_traces_dir:
         return Path(env_traces_dir)
-
-    # Default: ~/.agent-hippo/analytics/traces
     agent_home = Path.home() / '.agent-hippo'
     return agent_home / 'analytics' / 'traces'
 
-
-def get_db_path(traces_dir: Path, arg_db_path: str | None = None) -> Path:
-    """Get database path from arg, env var, or default (traces_dir/analytics.duckdb)."""
+def get_db_path(traces_dir: Path, arg_db_path: str | None=None) -> Path:
     if arg_db_path:
         return Path(arg_db_path)
-
     env_db_path = os.environ.get('AGENTIDE_INDEX_DB_PATH', '').strip()
     if env_db_path:
         return Path(env_db_path)
-
     return traces_dir / 'analytics.duckdb'
 
-
 def get_state_file(traces_dir: Path) -> Path:
-    """Get indexing state file path."""
     return traces_dir / '.duckdb-index.json'
 
-
 def get_lock_file(traces_dir: Path) -> Path:
-    """Get process lock file path."""
     return traces_dir / '.indexing.lock'
 
-
 def try_acquire_index_lock(lock_file: Path):
-    """
-    Try to acquire a cross-process exclusive lock.
-    Returns an open file handle when acquired, otherwise None.
-    """
     lock_file.parent.mkdir(parents=True, exist_ok=True)
     handle = lock_file.open('a+b')
-
-    # Ensure the file has at least one byte for Windows byte-range locking.
     handle.seek(0, os.SEEK_END)
     if handle.tell() == 0:
         handle.write(b'0')
         handle.flush()
-
     try:
         if os.name == 'nt':
             import msvcrt
@@ -102,21 +54,15 @@ def try_acquire_index_lock(lock_file: Path):
     except OSError:
         handle.close()
         return None
-
-    # Record owner PID for diagnostics.
     handle.seek(0)
     handle.truncate(0)
-    handle.write(f"{os.getpid()}\n".encode('utf-8'))
+    handle.write(f'{os.getpid()}\n'.encode('utf-8'))
     handle.flush()
-
     return handle
 
-
 def release_index_lock(handle) -> None:
-    """Release the process lock (best effort)."""
     if handle is None:
         return
-
     try:
         if os.name == 'nt':
             import msvcrt
@@ -130,34 +76,22 @@ def release_index_lock(handle) -> None:
     finally:
         handle.close()
 
-
 def load_index_state(state_file: Path) -> dict[str, Any]:
-    """Load indexing state from disk."""
     try:
         if state_file.exists():
             content = state_file.read_text('utf-8')
             return json.loads(content)
     except Exception as e:
-        log.debug(f"Failed to load state file: {e}")
-
-    return {
-        'version': 2,
-        'indexed': False,
-        'lastMtime': 0,
-        'duckdbPath': '',
-    }
-
+        log.debug(f'Failed to load state file: {e}')
+    return {'version': 2, 'indexed': False, 'lastMtime': 0, 'duckdbPath': ''}
 
 def save_index_state(state_file: Path, state: dict[str, Any]) -> None:
-    """Save indexing state to disk."""
     try:
         state_file.write_text(json.dumps(state, indent=2), 'utf-8')
     except Exception as e:
-        log.warning(f"Failed to save state file: {e}")
-
+        log.warning(f'Failed to save state file: {e}')
 
 def get_latest_jsonl_mtime(traces_dir: Path) -> int:
-    """Get maximum mtime of all JSONL files in traces_dir."""
     try:
         max_mtime = 0
         for jsonl_file in traces_dir.glob('*.jsonl'):
@@ -168,259 +102,71 @@ def get_latest_jsonl_mtime(traces_dir: Path) -> int:
                 pass
         return max_mtime
     except Exception as e:
-        log.error(f"Failed to get JSONL mtimes: {e}")
+        log.error(f'Failed to get JSONL mtimes: {e}')
         return 0
 
-
 def ensure_schema(conn) -> None:
-    """Create normalized LLM-agent analytics tables, marts, and indexes."""
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS model_calls (
-        model_call_id VARCHAR PRIMARY KEY,
-        trace_id VARCHAR,
-        span_id VARCHAR,
-        span_name VARCHAR,
-        session_id VARCHAR,
-        turn_id VARCHAR,
-        request_id VARCHAR,
-        user_id VARCHAR,
-        project_id VARCHAR,
-        agent_id VARCHAR,
-        agent_version VARCHAR,
-        source VARCHAR,
-        engine VARCHAR,
-        model VARCHAR,
-        start_ms BIGINT,
-        end_ms BIGINT,
-        duration_ms DOUBLE,
-        prompt_tokens BIGINT,
-        completion_tokens BIGINT,
-        total_tokens BIGINT,
-        total_cost_usd DOUBLE,
-        status VARCHAR,
-        error_type VARCHAR,
-        error_message VARCHAR,
-        input_messages_raw VARCHAR,
-        output_messages_raw VARCHAR,
-        created_at TIMESTAMP,
-        updated_at TIMESTAMP
-    )
-    """)
-
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS sessions (
-        session_id VARCHAR PRIMARY KEY,
-        user_id VARCHAR,
-        project_id VARCHAR,
-        agent_id VARCHAR,
-        agent_version VARCHAR,
-        source VARCHAR,
-        engine VARCHAR,
-        started_at TIMESTAMP,
-        ended_at TIMESTAMP,
-        status VARCHAR,
-        total_turns BIGINT,
-        total_requests BIGINT,
-        total_cost_usd DOUBLE,
-        total_tokens BIGINT,
-        created_at TIMESTAMP,
-        updated_at TIMESTAMP
-    )
-    """)
-
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS turns (
-        turn_id VARCHAR PRIMARY KEY,
-        session_id VARCHAR,
-        request_id VARCHAR,
-        turn_index BIGINT,
-        started_at TIMESTAMP,
-        ended_at TIMESTAMP,
-        status VARCHAR,
-        model VARCHAR,
-        provider VARCHAR,
-        prompt_tokens BIGINT,
-        completion_tokens BIGINT,
-        total_tokens BIGINT,
-        cost_usd DOUBLE,
-        trace_id VARCHAR,
-        created_at TIMESTAMP,
-        updated_at TIMESTAMP
-    )
-    """)
-
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS tool_calls (
-        tool_call_id VARCHAR PRIMARY KEY,
-        session_id VARCHAR,
-        turn_id VARCHAR,
-        request_id VARCHAR,
-        model_call_id VARCHAR,
-        trace_id VARCHAR,
-        span_id VARCHAR,
-        tool_name VARCHAR,
-        arguments_json JSON,
-        arguments_text VARCHAR,
-        call_ts TIMESTAMP,
-        result_json JSON,
-        result_text VARCHAR,
-        status VARCHAR,
-        duration_ms DOUBLE,
-        error_type VARCHAR,
-        error_message VARCHAR,
-        created_at TIMESTAMP,
-        updated_at TIMESTAMP
-    )
-    """)
-
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS artifacts (
-        artifact_id VARCHAR PRIMARY KEY,
-        session_id VARCHAR,
-        turn_id VARCHAR,
-        trace_id VARCHAR,
-        kind VARCHAR,
-        mime_type VARCHAR,
-        storage_uri VARCHAR,
-        sha256 VARCHAR,
-        byte_size BIGINT,
-        created_at TIMESTAMP
-    )
-    """)
-
-    # Gold compatibility/fast dashboard table
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS spans (
-        trace_id VARCHAR,
-        span_id VARCHAR,
-        span_name VARCHAR,
-        model VARCHAR,
-        cost DOUBLE,
-        input_tokens BIGINT,
-        output_tokens BIGINT,
-        session_id VARCHAR,
-        user_id VARCHAR,
-        project_id VARCHAR,
-        agent_id VARCHAR,
-        engine VARCHAR,
-        duration_ms DOUBLE,
-        start_ms BIGINT,
-        created_at TIMESTAMP,
-        PRIMARY KEY(trace_id, span_id)
-    )
-    """)
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS model_calls (\n        model_call_id VARCHAR PRIMARY KEY,\n        trace_id VARCHAR,\n        span_id VARCHAR,\n        span_name VARCHAR,\n        session_id VARCHAR,\n        turn_id VARCHAR,\n        request_id VARCHAR,\n        user_id VARCHAR,\n        project_id VARCHAR,\n        agent_id VARCHAR,\n        agent_version VARCHAR,\n        source VARCHAR,\n        engine VARCHAR,\n        model VARCHAR,\n        start_ms BIGINT,\n        end_ms BIGINT,\n        duration_ms DOUBLE,\n        prompt_tokens BIGINT,\n        completion_tokens BIGINT,\n        total_tokens BIGINT,\n        total_cost_usd DOUBLE,\n        status VARCHAR,\n        error_type VARCHAR,\n        error_message VARCHAR,\n        input_messages_raw VARCHAR,\n        output_messages_raw VARCHAR,\n        created_at TIMESTAMP,\n        updated_at TIMESTAMP\n    )\n    ')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS sessions (\n        session_id VARCHAR PRIMARY KEY,\n        user_id VARCHAR,\n        project_id VARCHAR,\n        agent_id VARCHAR,\n        agent_version VARCHAR,\n        source VARCHAR,\n        engine VARCHAR,\n        started_at TIMESTAMP,\n        ended_at TIMESTAMP,\n        status VARCHAR,\n        total_turns BIGINT,\n        total_requests BIGINT,\n        total_cost_usd DOUBLE,\n        total_tokens BIGINT,\n        created_at TIMESTAMP,\n        updated_at TIMESTAMP\n    )\n    ')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS turns (\n        turn_id VARCHAR PRIMARY KEY,\n        session_id VARCHAR,\n        request_id VARCHAR,\n        turn_index BIGINT,\n        started_at TIMESTAMP,\n        ended_at TIMESTAMP,\n        status VARCHAR,\n        model VARCHAR,\n        provider VARCHAR,\n        prompt_tokens BIGINT,\n        completion_tokens BIGINT,\n        total_tokens BIGINT,\n        cost_usd DOUBLE,\n        trace_id VARCHAR,\n        created_at TIMESTAMP,\n        updated_at TIMESTAMP\n    )\n    ')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS tool_calls (\n        tool_call_id VARCHAR PRIMARY KEY,\n        session_id VARCHAR,\n        turn_id VARCHAR,\n        request_id VARCHAR,\n        model_call_id VARCHAR,\n        trace_id VARCHAR,\n        span_id VARCHAR,\n        tool_name VARCHAR,\n        arguments_json JSON,\n        arguments_text VARCHAR,\n        call_ts TIMESTAMP,\n        result_json JSON,\n        result_text VARCHAR,\n        status VARCHAR,\n        duration_ms DOUBLE,\n        error_type VARCHAR,\n        error_message VARCHAR,\n        created_at TIMESTAMP,\n        updated_at TIMESTAMP\n    )\n    ')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS artifacts (\n        artifact_id VARCHAR PRIMARY KEY,\n        session_id VARCHAR,\n        turn_id VARCHAR,\n        trace_id VARCHAR,\n        kind VARCHAR,\n        mime_type VARCHAR,\n        storage_uri VARCHAR,\n        sha256 VARCHAR,\n        byte_size BIGINT,\n        created_at TIMESTAMP\n    )\n    ')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS spans (\n        trace_id VARCHAR,\n        span_id VARCHAR,\n        span_name VARCHAR,\n        model VARCHAR,\n        cost DOUBLE,\n        input_tokens BIGINT,\n        output_tokens BIGINT,\n        session_id VARCHAR,\n        user_id VARCHAR,\n        project_id VARCHAR,\n        agent_id VARCHAR,\n        engine VARCHAR,\n        duration_ms DOUBLE,\n        start_ms BIGINT,\n        created_at TIMESTAMP,\n        PRIMARY KEY(trace_id, span_id)\n    )\n    ')
     try:
-        conn.execute("ALTER TABLE spans ADD COLUMN IF NOT EXISTS engine VARCHAR")
+        conn.execute('ALTER TABLE spans ADD COLUMN IF NOT EXISTS engine VARCHAR')
     except Exception as e:
-        log.debug(f"Spans engine column (may already exist): {e}")
-
-    conn.execute("""
-    CREATE TABLE IF NOT EXISTS processed_files (
-        file_path VARCHAR PRIMARY KEY,
-        mtime_ms BIGINT NOT NULL,
-        size_bytes BIGINT NOT NULL,
-        processed_at TIMESTAMP NOT NULL
-    )
-    """)
-
-    conn.execute("""
-    CREATE OR REPLACE VIEW agent_kpis_daily AS
-    SELECT
-        strftime(to_timestamp(start_ms / 1000.0), '%Y-%m-%d') AS day,
-        coalesce(agent_id, '(no agent)') AS agent_id,
-        coalesce(agent_version, '(unknown)') AS agent_version,
-        coalesce(model, '(unknown)') AS model,
-        count(*) AS requests,
-        count(distinct session_id) AS sessions,
-        count(distinct turn_id) AS turns,
-        round(sum(coalesce(total_cost_usd, 0.0)), 6) AS total_cost_usd,
-        sum(coalesce(total_tokens, 0)) AS total_tokens,
-        round(avg(duration_ms), 2) AS avg_latency_ms,
-        round(percentile_cont(0.50) within group (order by duration_ms), 2) AS p50_latency_ms,
-        round(percentile_cont(0.95) within group (order by duration_ms), 2) AS p95_latency_ms,
-        round(100.0 * sum(case when status = 'error' then 1 else 0 end) / nullif(count(*), 0), 2) AS error_rate_pct
-    FROM model_calls
-    GROUP BY 1, 2, 3, 4
-    """)
-
-    # Common indexes for dashboard and skill queries.
-    for stmt in [
-        "CREATE INDEX IF NOT EXISTS idx_model_calls_session_id ON model_calls(session_id)",
-        "CREATE INDEX IF NOT EXISTS idx_model_calls_start_ms ON model_calls(start_ms)",
-        "CREATE INDEX IF NOT EXISTS idx_model_calls_agent_id ON model_calls(agent_id)",
-        "CREATE INDEX IF NOT EXISTS idx_model_calls_project_id ON model_calls(project_id)",
-        "CREATE INDEX IF NOT EXISTS idx_tool_calls_session ON tool_calls(session_id)",
-        "CREATE INDEX IF NOT EXISTS idx_spans_session ON spans(session_id)",
-        "CREATE INDEX IF NOT EXISTS idx_spans_start_ms ON spans(start_ms)",
-        "CREATE INDEX IF NOT EXISTS idx_spans_model ON spans(model)",
-        "CREATE INDEX IF NOT EXISTS idx_spans_agent_id ON spans(agent_id)",
-        "CREATE INDEX IF NOT EXISTS idx_spans_project_id ON spans(project_id)",
-    ]:
+        log.debug(f'Spans engine column (may already exist): {e}')
+    conn.execute('\n    CREATE TABLE IF NOT EXISTS processed_files (\n        file_path VARCHAR PRIMARY KEY,\n        mtime_ms BIGINT NOT NULL,\n        size_bytes BIGINT NOT NULL,\n        processed_at TIMESTAMP NOT NULL\n    )\n    ')
+    conn.execute("\n    CREATE OR REPLACE VIEW agent_kpis_daily AS\n    SELECT\n        strftime(to_timestamp(start_ms / 1000.0), '%Y-%m-%d') AS day,\n        coalesce(agent_id, '(no agent)') AS agent_id,\n        coalesce(agent_version, '(unknown)') AS agent_version,\n        coalesce(model, '(unknown)') AS model,\n        count(*) AS requests,\n        count(distinct session_id) AS sessions,\n        count(distinct turn_id) AS turns,\n        round(sum(coalesce(total_cost_usd, 0.0)), 6) AS total_cost_usd,\n        sum(coalesce(total_tokens, 0)) AS total_tokens,\n        round(avg(duration_ms), 2) AS avg_latency_ms,\n        round(percentile_cont(0.50) within group (order by duration_ms), 2) AS p50_latency_ms,\n        round(percentile_cont(0.95) within group (order by duration_ms), 2) AS p95_latency_ms,\n        round(100.0 * sum(case when status = 'error' then 1 else 0 end) / nullif(count(*), 0), 2) AS error_rate_pct\n    FROM model_calls\n    GROUP BY 1, 2, 3, 4\n    ")
+    for stmt in ['CREATE INDEX IF NOT EXISTS idx_model_calls_session_id ON model_calls(session_id)', 'CREATE INDEX IF NOT EXISTS idx_model_calls_start_ms ON model_calls(start_ms)', 'CREATE INDEX IF NOT EXISTS idx_model_calls_agent_id ON model_calls(agent_id)', 'CREATE INDEX IF NOT EXISTS idx_model_calls_project_id ON model_calls(project_id)', 'CREATE INDEX IF NOT EXISTS idx_tool_calls_session ON tool_calls(session_id)', 'CREATE INDEX IF NOT EXISTS idx_spans_session ON spans(session_id)', 'CREATE INDEX IF NOT EXISTS idx_spans_start_ms ON spans(start_ms)', 'CREATE INDEX IF NOT EXISTS idx_spans_model ON spans(model)', 'CREATE INDEX IF NOT EXISTS idx_spans_agent_id ON spans(agent_id)', 'CREATE INDEX IF NOT EXISTS idx_spans_project_id ON spans(project_id)']:
         try:
             conn.execute(stmt)
         except Exception as e:
-            log.debug(f"Index creation hint (may already exist): {e}")
-
+            log.debug(f'Index creation hint (may already exist): {e}')
 
 def get_file_stats(jsonl_file: Path) -> tuple[int, int]:
     stats = jsonl_file.stat()
-    return int(stats.st_mtime * 1000), int(stats.st_size)
-
+    return (int(stats.st_mtime * 1000), int(stats.st_size))
 
 def get_processed_file_map(conn) -> dict[str, tuple[int, int]]:
-    rows = conn.execute(
-        "SELECT file_path, mtime_ms, size_bytes FROM processed_files"
-    ).fetchall()
+    rows = conn.execute('SELECT file_path, mtime_ms, size_bytes FROM processed_files').fetchall()
     return {row[0]: (int(row[1]), int(row[2])) for row in rows}
 
-
 def get_changed_files(conn, traces_dir: Path) -> list[Path]:
     processed = get_processed_file_map(conn)
     changed: list[Path] = []
-
     for jsonl_file in sorted(traces_dir.glob('*.jsonl')):
         try:
             mtime_ms, size_bytes = get_file_stats(jsonl_file)
         except OSError:
             continue
-
         key = str(jsonl_file)
         previous = processed.get(key)
         if previous is None or previous[0] != mtime_ms or previous[1] != size_bytes:
             changed.append(jsonl_file)
-
     return changed
 
-
 def _safe_json_parse(raw: Any) -> Any:
     if raw is None:
         return None
     if isinstance(raw, (dict, list)):
         return raw
-
     text = str(raw).strip()
     if text == '':
         return None
-
-    # try strict JSON first
     try:
         parsed = json.loads(text)
     except Exception:
         parsed = None
-
-    # fallback to python-literal-like payloads (single-quoted dict/list)
     if parsed is None:
         try:
             parsed = ast.literal_eval(text)
         except Exception:
             parsed = None
-
     if isinstance(parsed, str) and parsed != text:
         return _safe_json_parse(parsed)
-
     return parsed
 
-
 def _to_json_text(value: Any) -> str | None:
     if value is None:
         return None
@@ -429,7 +175,6 @@ def _to_json_text(value: Any) -> str | None:
     except Exception:
         return None
 
-
 def _extract_text_content(content: Any) -> str:
     if content is None:
         return ''
@@ -460,13 +205,10 @@ def _extract_text_content(content: Any) -> str:
             return _extract_text_content(content.get('input'))
     return str(content)
 
-
 def _normalize_messages(payload: Any, default_role: str) -> list[dict[str, Any]]:
     parsed = _safe_json_parse(payload)
-
     if parsed is None:
         return []
-
     if isinstance(parsed, dict):
         if isinstance(parsed.get('messages'), list):
             parsed = parsed['messages']
@@ -480,81 +222,29 @@ def _normalize_messages(payload: Any, default_role: str) -> list[dict[str, Any]]
             return out
         else:
             parsed = [parsed]
-
     if not isinstance(parsed, list):
-        return [{"role": default_role, "content": str(parsed)}]
-
+        return [{'role': default_role, 'content': str(parsed)}]
     out: list[dict[str, Any]] = []
     for entry in parsed:
         if isinstance(entry, dict):
             out.append(entry)
         else:
-            out.append({"role": default_role, "content": str(entry)})
+            out.append({'role': default_role, 'content': str(entry)})
     return out
 
-
 def _tool_call_id(session_id: str | None, trace_id: str, call_id: str | None, idx: int) -> str:
     if call_id:
         return call_id
-    seed = f"{session_id or ''}|{trace_id}|tool|{idx}"
+    seed = f'{session_id or ''}|{trace_id}|tool|{idx}'
     return hashlib.sha1(seed.encode('utf-8')).hexdigest()
 
-
 def extract_messages_and_tools(conn) -> None:
-    """
-    Parse tool-call payloads from current temp batch (tmp_model_calls)
-    and upsert into tool_calls.
-    """
-    rows = conn.execute(
-        """
-        SELECT
-            model_call_id,
-            trace_id,
-            span_id,
-            session_id,
-            turn_id,
-            request_id,
-            start_ms,
-            end_ms,
-            status,
-            error_type,
-            error_message,
-            input_messages_raw,
-            output_messages_raw,
-            span_tool_name,
-            span_tool_call_id,
-            span_tool_arguments_raw,
-            span_tool_result_raw
-        FROM tmp_model_calls
-        """
-    ).fetchall()
-
+    rows = conn.execute('\n        SELECT\n            model_call_id,\n            trace_id,\n            span_id,\n            session_id,\n            turn_id,\n            request_id,\n            start_ms,\n            end_ms,\n            status,\n            error_type,\n            error_message,\n            input_messages_raw,\n            output_messages_raw,\n            span_tool_name,\n            span_tool_call_id,\n            span_tool_arguments_raw,\n            span_tool_result_raw\n        FROM tmp_model_calls\n        ').fetchall()
     tool_results_by_id: dict[str, dict[str, Any]] = {}
-
     for row in rows:
-        (
-            model_call_id,
-            trace_id,
-            span_id,
-            session_id,
-            turn_id,
-            request_id,
-            start_ms,
-            end_ms,
-            call_status,
-            call_error_type,
-            call_error_message,
-            input_raw,
-            output_raw,
-            span_tool_name,
-            span_tool_call_id,
-            span_tool_arguments_raw,
-            span_tool_result_raw,
-        ) = row
-
+        model_call_id, trace_id, span_id, session_id, turn_id, request_id, start_ms, end_ms, call_status, call_error_type, call_error_message, input_raw, output_raw, span_tool_name, span_tool_call_id, span_tool_arguments_raw, span_tool_result_raw = row
         tool_call_inserted = False
         tool_base_ts = int(end_ms) if end_ms is not None else int(start_ms) if start_ms is not None else 0
-
         if span_tool_name:
             fallback_call_id = span_tool_call_id
             if fallback_call_id is not None:
@@ -565,106 +255,17 @@ def extract_messages_and_tools(conn) -> None:
             fallback_result_json = _to_json_text(_safe_json_parse(span_tool_result_raw) if span_tool_result_raw else None)
             if fallback_result_json is None and span_tool_result_raw:
                 fallback_result_json = _to_json_text(span_tool_result_raw)
-
-            conn.execute(
-                """
-                INSERT INTO tool_calls (
-                    tool_call_id, session_id, turn_id, request_id, model_call_id,
-                    trace_id, span_id, tool_name, arguments_json, arguments_text,
-                    call_ts, result_json, result_text, status, duration_ms,
-                    error_type, error_message, created_at, updated_at
-                )
-                VALUES (
-                    ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,
-                    to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,
-                    ?, ?, now(), now()
-                )
-                ON CONFLICT(tool_call_id) DO UPDATE SET
-                    result_json = COALESCE(excluded.result_json, tool_calls.result_json),
-                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),
-                    status = CASE
-                        WHEN excluded.status = 'error' THEN 'error'
-                        WHEN excluded.status = 'ok' THEN 'ok'
-                        ELSE tool_calls.status
-                    END,
-                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),
-                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),
-                    updated_at = now()
-                """,
-                [
-                    fallback_tool_call_id,
-                    session_id,
-                    turn_id,
-                    request_id,
-                    model_call_id,
-                    trace_id,
-                    span_id,
-                    str(span_tool_name),
-                    _to_json_text(_safe_json_parse(span_tool_arguments_raw) if span_tool_arguments_raw else None)
-                    or _to_json_text(span_tool_arguments_raw),
-                    _extract_text_content(span_tool_arguments_raw),
-                    tool_base_ts,
-                    fallback_result_json,
-                    fallback_result_text,
-                    fallback_status,
-                    call_error_type,
-                    call_error_message,
-                ],
-            )
+            conn.execute("\n                INSERT INTO tool_calls (\n                    tool_call_id, session_id, turn_id, request_id, model_call_id,\n                    trace_id, span_id, tool_name, arguments_json, arguments_text,\n                    call_ts, result_json, result_text, status, duration_ms,\n                    error_type, error_message, created_at, updated_at\n                )\n                VALUES (\n                    ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,\n                    to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,\n                    ?, ?, now(), now()\n                )\n                ON CONFLICT(tool_call_id) DO UPDATE SET\n                    result_json = COALESCE(excluded.result_json, tool_calls.result_json),\n                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),\n                    status = CASE\n                        WHEN excluded.status = 'error' THEN 'error'\n                        WHEN excluded.status = 'ok' THEN 'ok'\n                        ELSE tool_calls.status\n                    END,\n                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),\n                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),\n                    updated_at = now()\n                ", [fallback_tool_call_id, session_id, turn_id, request_id, model_call_id, trace_id, span_id, str(span_tool_name), _to_json_text(_safe_json_parse(span_tool_arguments_raw) if span_tool_arguments_raw else None) or _to_json_text(span_tool_arguments_raw), _extract_text_content(span_tool_arguments_raw), tool_base_ts, fallback_result_json, fallback_result_text, fallback_status, call_error_type, call_error_message])
             tool_call_inserted = True
-
-        if str(call_status or '').lower() == 'error' and not tool_call_inserted and call_error_message:
+        if str(call_status or '').lower() == 'error' and (not tool_call_inserted) and call_error_message:
             synthetic_tool_call_id = _tool_call_id(session_id, str(trace_id), None, 0)
-            conn.execute(
-                """
-                INSERT INTO tool_calls (
-                    tool_call_id, session_id, turn_id, request_id, model_call_id,
-                    trace_id, span_id, tool_name, arguments_json, arguments_text,
-                    call_ts, result_json, result_text, status, duration_ms,
-                    error_type, error_message, created_at, updated_at
-                )
-                VALUES (
-                    ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL,
-                    to_timestamp(? / 1000.0), NULL, ?, 'error', NULL,
-                    ?, ?, now(), now()
-                )
-                ON CONFLICT(tool_call_id) DO UPDATE SET
-                    status = 'error',
-                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),
-                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),
-                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),
-                    updated_at = now()
-                """,
-                [
-                    synthetic_tool_call_id,
-                    session_id,
-                    turn_id,
-                    request_id,
-                    model_call_id,
-                    trace_id,
-                    span_id,
-                    '(tool-error)',
-                    tool_base_ts,
-                    call_error_message,
-                    call_error_type,
-                    call_error_message,
-                ],
-            )
-
+            conn.execute("\n                INSERT INTO tool_calls (\n                    tool_call_id, session_id, turn_id, request_id, model_call_id,\n                    trace_id, span_id, tool_name, arguments_json, arguments_text,\n                    call_ts, result_json, result_text, status, duration_ms,\n                    error_type, error_message, created_at, updated_at\n                )\n                VALUES (\n                    ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL,\n                    to_timestamp(? / 1000.0), NULL, ?, 'error', NULL,\n                    ?, ?, now(), now()\n                )\n                ON CONFLICT(tool_call_id) DO UPDATE SET\n                    status = 'error',\n                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),\n                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),\n                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),\n                    updated_at = now()\n                ", [synthetic_tool_call_id, session_id, turn_id, request_id, model_call_id, trace_id, span_id, '(tool-error)', tool_base_ts, call_error_message, call_error_type, call_error_message])
         if span_tool_call_id:
-            tool_results_by_id[str(span_tool_call_id)] = {
-                'result_json': _to_json_text(_safe_json_parse(span_tool_result_raw) if span_tool_result_raw else None),
-                'result_text': _extract_text_content(span_tool_result_raw),
-                'status': 'error' if str(call_status or '').lower() == 'error' else 'ok',
-                'updated_at_ms': tool_base_ts,
-            }
+            tool_results_by_id[str(span_tool_call_id)] = {'result_json': _to_json_text(_safe_json_parse(span_tool_result_raw) if span_tool_result_raw else None), 'result_text': _extract_text_content(span_tool_result_raw), 'status': 'error' if str(call_status or '').lower() == 'error' else 'ok', 'updated_at_ms': tool_base_ts}
             if tool_results_by_id[str(span_tool_call_id)]['result_json'] is None and span_tool_result_raw:
                 tool_results_by_id[str(span_tool_call_id)]['result_json'] = _to_json_text(span_tool_result_raw)
-
         input_messages = _normalize_messages(input_raw, 'user')
         output_messages = _normalize_messages(output_raw, 'assistant')
-
-        # tool_result blocks may appear in input messages and complete tool calls emitted in output
         for i, msg in enumerate(input_messages):
             content = msg.get('content', msg.get('parts', msg.get('text', '')))
             if isinstance(content, list):
@@ -677,23 +278,13 @@ def extract_messages_and_tools(conn) -> None:
                     call_id = block.get('tool_use_id') or block.get('call_id') or block.get('id')
                     if not call_id:
                         continue
-                    tool_results_by_id[str(call_id)] = {
-                        'result_json': _to_json_text(block.get('content', block.get('result'))),
-                        'result_text': _extract_text_content(block.get('content', block.get('result'))),
-                        'status': 'ok',
-                        'updated_at_ms': int(start_ms) + j if start_ms is not None else 0,
-                    }
-
-        # Output-side tool calls
+                    tool_results_by_id[str(call_id)] = {'result_json': _to_json_text(block.get('content', block.get('result'))), 'result_text': _extract_text_content(block.get('content', block.get('result'))), 'status': 'ok', 'updated_at_ms': int(start_ms) + j if start_ms is not None else 0}
         for i, msg in enumerate(output_messages):
             content = msg.get('content', msg.get('parts', msg.get('text', '')))
             output_ts = int(end_ms) if end_ms is not None else int(start_ms) if start_ms is not None else 0
-
-            # Normalize tool blocks from multiple provider formats.
             tool_blocks: list[dict[str, Any]] = []
             if isinstance(content, list):
                 tool_blocks.extend([b for b in content if isinstance(b, dict)])
-
             direct_tool_calls = msg.get('tool_calls')
             if isinstance(direct_tool_calls, list):
                 for block in direct_tool_calls:
@@ -701,635 +292,108 @@ def extract_messages_and_tools(conn) -> None:
                         normalized = dict(block)
                         normalized.setdefault('type', 'tool_call')
                         tool_blocks.append(normalized)
-
             function_call = msg.get('function_call')
             if isinstance(function_call, dict):
                 tool_blocks.append({'type': 'function_call', 'function': function_call})
-
             if not tool_blocks:
                 continue
-
             for j, block in enumerate(tool_blocks):
                 if not isinstance(block, dict):
                     continue
-
                 block_type = str(block.get('type', '')).lower()
                 if block_type not in {'tool_use', 'tool_call', 'function_call', 'server_tool_use', 'mcp_tool_use'}:
                     continue
-
-                if block_type in {'server_tool_use', 'mcp_tool_use'} and not block.get('name'):
-                    # Normalize Anthropic streaming variants that still represent tool invocation.
+                if block_type in {'server_tool_use', 'mcp_tool_use'} and (not block.get('name')):
                     block['name'] = block.get('tool_name') or block.get('toolName')
                     block.setdefault('input', block.get('arguments'))
-
-                if block_type == 'tool_call' and not block.get('function') and isinstance(block.get('name'), str):
-                    block['function'] = {
-                        'name': block.get('name'),
-                        'arguments': block.get('arguments') or block.get('input'),
-                    }
-
-                if block_type == 'function_call' and not block.get('function') and (block.get('name') or block.get('arguments')):
-                    block['function'] = {
-                        'name': block.get('name'),
-                        'arguments': block.get('arguments'),
-                    }
-
-                if block_type == 'tool_use' and not block.get('name'):
+                if block_type == 'tool_call' and (not block.get('function')) and isinstance(block.get('name'), str):
+                    block['function'] = {'name': block.get('name'), 'arguments': block.get('arguments') or block.get('input')}
+                if block_type == 'function_call' and (not block.get('function')) and (block.get('name') or block.get('arguments')):
+                    block['function'] = {'name': block.get('name'), 'arguments': block.get('arguments')}
+                if block_type == 'tool_use' and (not block.get('name')):
                     block['name'] = block.get('tool_name') or block.get('toolName')
-
-                if block_type == 'tool_use' and block.get('input') is None and block.get('arguments') is not None:
+                if block_type == 'tool_use' and block.get('input') is None and (block.get('arguments') is not None):
                     block['input'] = block.get('arguments')
-
                 if block_type in {'tool_call', 'function_call'} and block.get('function') is None and block.get('name'):
-                    block['function'] = {
-                        'name': block.get('name'),
-                        'arguments': block.get('arguments') or block.get('input'),
-                    }
-
+                    block['function'] = {'name': block.get('name'), 'arguments': block.get('arguments') or block.get('input')}
                 if block_type == 'function_call' and block.get('function') is None:
                     continue
-
-                if block_type == 'tool_call' and block.get('function') is None and block.get('name') is None:
+                if block_type == 'tool_call' and block.get('function') is None and (block.get('name') is None):
                     continue
-
-                if block_type in {'server_tool_use', 'mcp_tool_use'} and not block.get('name'):
+                if block_type in {'server_tool_use', 'mcp_tool_use'} and (not block.get('name')):
                     continue
-
-                if block_type == 'tool_use' and not block.get('name'):
+                if block_type == 'tool_use' and (not block.get('name')):
                     continue
-
-                if block_type == 'tool_call' and block.get('function') is None and not block.get('name'):
+                if block_type == 'tool_call' and block.get('function') is None and (not block.get('name')):
                     continue
-
                 if block_type == 'function_call' and block.get('function') is None:
                     continue
-
-                # Harmonize final type for downstream extraction.
                 if block_type in {'server_tool_use', 'mcp_tool_use'}:
                     block_type = 'tool_use'
-
                 block['type'] = block_type
-
-
                 call_id = block.get('id') or block.get('tool_call_id') or block.get('call_id')
-                tool_name = (
-                    block.get('name')
-                    or (block.get('function', {}) or {}).get('name')
-                    or 'unknown_tool'
-                )
-
-                args = (
-                    block.get('input')
-                    if block.get('input') is not None
-                    else (block.get('function', {}) or {}).get('arguments')
-                )
+                tool_name = block.get('name') or (block.get('function', {}) or {}).get('name') or 'unknown_tool'
+                args = block.get('input') if block.get('input') is not None else (block.get('function', {}) or {}).get('arguments')
                 args_json = _to_json_text(args)
                 args_text = _extract_text_content(args)
-
                 tool_call_id = _tool_call_id(session_id, str(trace_id), str(call_id) if call_id else None, j)
                 tool_result = tool_results_by_id.get(str(call_id)) if call_id else None
-
                 result_json = tool_result['result_json'] if tool_result else None
                 result_text = tool_result['result_text'] if tool_result else None
                 status = tool_result['status'] if tool_result else 'called'
-
-                conn.execute(
-                    """
-                    INSERT INTO tool_calls (
-                        tool_call_id, session_id, turn_id, request_id, model_call_id,
-                        trace_id, span_id, tool_name, arguments_json, arguments_text,
-                        call_ts, result_json, result_text, status, duration_ms,
-                        error_type, error_message, created_at, updated_at
-                    )
-                    VALUES (
-                        ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,
-                        to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,
-                        NULL, NULL, now(), now()
-                    )
-                    ON CONFLICT(tool_call_id) DO UPDATE SET
-                        result_json = COALESCE(excluded.result_json, tool_calls.result_json),
-                        result_text = COALESCE(excluded.result_text, tool_calls.result_text),
-                        status = CASE
-                            WHEN excluded.status = 'ok' THEN 'ok'
-                            ELSE tool_calls.status
-                        END,
-                        updated_at = now()
-                    """,
-                    [
-                        tool_call_id,
-                        session_id,
-                        turn_id,
-                        request_id,
-                        model_call_id,
-                        trace_id,
-                        span_id,
-                        str(tool_name),
-                        args_json,
-                        args_text,
-                        output_ts + j,
-                        result_json,
-                        result_text,
-                        status,
-                    ],
-                )
-
+                conn.execute("\n                    INSERT INTO tool_calls (\n                        tool_call_id, session_id, turn_id, request_id, model_call_id,\n                        trace_id, span_id, tool_name, arguments_json, arguments_text,\n                        call_ts, result_json, result_text, status, duration_ms,\n                        error_type, error_message, created_at, updated_at\n                    )\n                    VALUES (\n                        ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,\n                        to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,\n                        NULL, NULL, now(), now()\n                    )\n                    ON CONFLICT(tool_call_id) DO UPDATE SET\n                        result_json = COALESCE(excluded.result_json, tool_calls.result_json),\n                        result_text = COALESCE(excluded.result_text, tool_calls.result_text),\n                        status = CASE\n                            WHEN excluded.status = 'ok' THEN 'ok'\n                            ELSE tool_calls.status\n                        END,\n                        updated_at = now()\n                    ", [tool_call_id, session_id, turn_id, request_id, model_call_id, trace_id, span_id, str(tool_name), args_json, args_text, output_ts + j, result_json, result_text, status])
 
 def ingest_single_jsonl_file(conn, jsonl_file: Path) -> None:
-    """Ingest one JSONL file into Silver and Gold analytics tables."""
     escaped = str(jsonl_file).replace("'", "''")
-
-    conn.execute("DROP TABLE IF EXISTS tmp_model_calls")
-    conn.execute(f"""
-    CREATE TEMP TABLE tmp_model_calls AS
-    WITH raw_spans AS (
-        SELECT unnest(resourceSpans) as rs
-        FROM read_json('{escaped}', format='newline_delimited', ignore_errors=true)
-    ),
-    spans_unnest AS (SELECT unnest(rs.scopeSpans) as ss FROM raw_spans),
-    flat_spans AS (SELECT unnest(ss.spans) as span FROM spans_unnest),
-    parsed AS (
-        SELECT
-            span.traceId as trace_id,
-            span.spanId as span_id,
-            span.name as span_name,
-            span.startTimeUnixNano::BIGINT / 1000000 as start_ms,
-            span.endTimeUnixNano::BIGINT / 1000000 as end_ms,
-            (span.endTimeUnixNano::BIGINT - span.startTimeUnixNano::BIGINT) / 1000000.0 as duration_ms,
-            json_extract(to_json(span), '$.attributes') as attrs_json
-        FROM flat_spans
-    ),
-    attrs AS (
-        SELECT
-            trace_id,
-            span_id,
-            span_name,
-            duration_ms,
-            start_ms,
-            end_ms,
-            json_extract_string(value, '$.key') as attr_key,
-            json_extract_string(value, '$.value.stringValue') as attr_string,
-            json_extract_string(value, '$.value.intValue') as attr_int,
-            json_extract(value, '$.value.doubleValue') as attr_double
-        FROM parsed, json_each(coalesce(attrs_json, '[]'::JSON))
-    ),
-    extracted AS (
-        SELECT
-            trace_id,
-            span_id,
-            span_name,
-            start_ms,
-            end_ms,
-            duration_ms,
-            max(CASE WHEN attr_key IN ('gen_ai.request.model','gen_ai.response.model','model') THEN attr_string END) as model,
-            max(CASE WHEN attr_key = 'gen_ai.cost.total_cost' THEN CAST(attr_double AS DOUBLE) END) as total_cost_usd,
-            max(CASE WHEN attr_key = 'gen_ai.usage.input_tokens' THEN CAST(attr_int AS BIGINT) END) as prompt_tokens,
-            max(CASE WHEN attr_key = 'gen_ai.usage.output_tokens' THEN CAST(attr_int AS BIGINT) END) as completion_tokens,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-session-id'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'session_id'': ''([^'']+)''', 1), '')
-            ) as session_id,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-user-id'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'user_id'': ''([^'']+)''', 1), '')
-            ) as user_id,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-project-id'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'project_id'': ''([^'']+)''', 1), '')
-            ) as project_id,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-id'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_id'': ''([^'']+)''', 1), '')
-            ) as agent_id,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-version'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_version'': ''([^'']+)''', 1), '')
-            ) as agent_version,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-source'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'source'': ''([^'']+)''', 1), '')
-            ) as source,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-engine'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'engine'': ''([^'']+)''', 1), '')
-            ) as engine,
-            COALESCE(
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-request-id'': ''([^'']+)''', 1), ''),
-                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'request_id'': ''([^'']+)''', 1), '')
-            ) as request_id,
-            max(CASE WHEN attr_key IN ('otel.status_code') THEN attr_string END) as otel_status,
-            max(CASE WHEN attr_key IN ('exception.type') THEN attr_string END) as error_type,
-            max(CASE WHEN attr_key IN ('exception.message') THEN attr_string END) as error_message,
-            max(CASE WHEN attr_key IN (
-                'gen_ai.tool.name',
-                'tool.name',
-                'toolName'
-            ) THEN attr_string END) as span_tool_name,
-            max(CASE WHEN attr_key IN (
-                'gen_ai.tool.call.id',
-                'tool.call.id',
-                'tool_call_id',
-                'tool.use.id',
-                'tool_use_id'
-            ) THEN attr_string END) as span_tool_call_id,
-            max(CASE WHEN attr_key IN (
-                'gen_ai.tool.arguments',
-                'tool.arguments',
-                'tool.input',
-                'tool.args'
-            ) THEN attr_string END) as span_tool_arguments_raw,
-            max(CASE WHEN attr_key IN (
-                'gen_ai.tool.result',
-                'tool.result',
-                'tool.output'
-            ) THEN attr_string END) as span_tool_result_raw,
-            max(CASE WHEN attr_key = 'gen_ai.input.messages' THEN attr_string END) as input_messages_raw,
-            max(CASE WHEN attr_key = 'gen_ai.output.messages' THEN attr_string END) as output_messages_raw
-        FROM attrs
-        GROUP BY trace_id, span_id, span_name, duration_ms, start_ms, end_ms
-    ),
-    representative AS (
-        SELECT DISTINCT ON (trace_id)
-            trace_id,
-            span_id,
-            span_name,
-            session_id,
-            request_id,
-            user_id,
-            project_id,
-            agent_id,
-            agent_version,
-            source,
-            engine,
-            model,
-            start_ms,
-            end_ms,
-            duration_ms,
-            COALESCE(prompt_tokens, 0) as prompt_tokens,
-            COALESCE(completion_tokens, 0) as completion_tokens,
-            COALESCE(total_cost_usd, 0.0) as total_cost_usd,
-            error_type,
-            error_message,
-            span_tool_name,
-            span_tool_call_id,
-            span_tool_arguments_raw,
-            span_tool_result_raw,
-            input_messages_raw,
-            output_messages_raw
-        FROM extracted
-        ORDER BY
-            trace_id,
-            CASE WHEN output_messages_raw IS NOT NULL AND output_messages_raw != '' THEN 0 ELSE 1 END,
-            CASE WHEN input_messages_raw IS NOT NULL AND input_messages_raw != '' THEN 0 ELSE 1 END,
-            COALESCE(total_cost_usd, 0) DESC,
-            CASE
-                WHEN span_name = 'raw_gen_ai_request' THEN 0
-                WHEN span_name = 'litellm_request' THEN 1
-                ELSE 2
-            END,
-            start_ms DESC
-    ),
-    merged AS (
-        SELECT
-            e.trace_id,
-            r.span_id,
-            r.span_name,
-            COALESCE(MAX(CASE WHEN e.session_id IS NOT NULL AND e.session_id != '' THEN e.session_id END), r.session_id) AS session_id,
-            COALESCE(MAX(CASE WHEN e.request_id IS NOT NULL AND e.request_id != '' THEN e.request_id END), r.request_id) AS request_id,
-            COALESCE(MAX(CASE WHEN e.user_id IS NOT NULL AND e.user_id != '' THEN e.user_id END), r.user_id) AS user_id,
-            COALESCE(MAX(CASE WHEN e.project_id IS NOT NULL AND e.project_id != '' THEN e.project_id END), r.project_id) AS project_id,
-            COALESCE(MAX(CASE WHEN e.agent_id IS NOT NULL AND e.agent_id != '' THEN e.agent_id END), r.agent_id) AS agent_id,
-            COALESCE(MAX(CASE WHEN e.agent_version IS NOT NULL AND e.agent_version != '' THEN e.agent_version END), r.agent_version) AS agent_version,
-            COALESCE(MAX(CASE WHEN e.source IS NOT NULL AND e.source != '' THEN e.source END), r.source) AS source,
-            COALESCE(MAX(CASE WHEN e.engine IS NOT NULL AND e.engine != '' THEN e.engine END), r.engine) AS engine,
-            COALESCE(MAX(CASE WHEN e.model IS NOT NULL AND e.model != '' THEN e.model END), r.model) AS model,
-            MIN(e.start_ms) AS start_ms,
-            MAX(e.end_ms) AS end_ms,
-            MAX(e.duration_ms) AS duration_ms,
-            COALESCE(MAX(e.prompt_tokens), r.prompt_tokens, 0) AS prompt_tokens,
-            COALESCE(MAX(e.completion_tokens), r.completion_tokens, 0) AS completion_tokens,
-            COALESCE(MAX(e.total_cost_usd), r.total_cost_usd, 0.0) AS total_cost_usd,
-            CASE
-                WHEN SUM(CASE WHEN lower(coalesce(e.otel_status, '')) = 'error' OR e.error_type IS NOT NULL OR e.error_message IS NOT NULL THEN 1 ELSE 0 END) > 0 THEN 'error'
-                ELSE 'ok'
-            END AS status,
-            COALESCE(MAX(CASE WHEN e.error_type IS NOT NULL AND e.error_type != '' THEN e.error_type END), r.error_type) AS error_type,
-            COALESCE(MAX(CASE WHEN e.error_message IS NOT NULL AND e.error_message != '' THEN e.error_message END), r.error_message) AS error_message,
-            COALESCE(MAX(CASE WHEN e.span_tool_name IS NOT NULL AND e.span_tool_name != '' THEN e.span_tool_name END), r.span_tool_name) AS span_tool_name,
-            COALESCE(MAX(CASE WHEN e.span_tool_call_id IS NOT NULL AND e.span_tool_call_id != '' THEN e.span_tool_call_id END), r.span_tool_call_id) AS span_tool_call_id,
-            COALESCE(MAX(CASE WHEN e.span_tool_arguments_raw IS NOT NULL AND e.span_tool_arguments_raw != '' THEN e.span_tool_arguments_raw END), r.span_tool_arguments_raw) AS span_tool_arguments_raw,
-            COALESCE(MAX(CASE WHEN e.span_tool_result_raw IS NOT NULL AND e.span_tool_result_raw != '' THEN e.span_tool_result_raw END), r.span_tool_result_raw) AS span_tool_result_raw,
-            COALESCE(MAX(CASE WHEN e.input_messages_raw IS NOT NULL AND e.input_messages_raw != '' THEN e.input_messages_raw END), r.input_messages_raw) AS input_messages_raw,
-            COALESCE(MAX(CASE WHEN e.output_messages_raw IS NOT NULL AND e.output_messages_raw != '' THEN e.output_messages_raw END), r.output_messages_raw) AS output_messages_raw
-        FROM extracted e
-        JOIN representative r ON r.trace_id = e.trace_id
-        GROUP BY e.trace_id, r.span_id, r.span_name, r.session_id, r.request_id, r.user_id, r.project_id, r.agent_id, r.agent_version, r.source, r.engine, r.model, r.prompt_tokens, r.completion_tokens, r.total_cost_usd, r.error_type, r.error_message, r.span_tool_name, r.span_tool_call_id, r.span_tool_arguments_raw, r.span_tool_result_raw, r.input_messages_raw, r.output_messages_raw
-    )
-    SELECT
-        trace_id as model_call_id,
-        trace_id,
-        span_id,
-        span_name,
-        session_id,
-        CASE
-            WHEN session_id IS NOT NULL AND request_id IS NOT NULL THEN session_id || ':' || request_id
-            ELSE trace_id
-        END as turn_id,
-        request_id,
-        user_id,
-        project_id,
-        agent_id,
-        agent_version,
-        source,
-        engine,
-        model,
-        start_ms,
-        end_ms,
-        duration_ms,
-        prompt_tokens,
-        completion_tokens,
-        COALESCE(prompt_tokens, 0) + COALESCE(completion_tokens, 0) as total_tokens,
-        total_cost_usd,
-        status,
-        error_type,
-        error_message,
-        span_tool_name,
-        span_tool_call_id,
-        span_tool_arguments_raw,
-        span_tool_result_raw,
-        input_messages_raw,
-        output_messages_raw
-    FROM merged
-    """)
-
-    # Silver: model calls
-    conn.execute("""
-    INSERT INTO model_calls (
-        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,
-        user_id, project_id, agent_id, agent_version, source, engine, model,
-        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,
-        total_cost_usd, status, error_type, error_message,
-        input_messages_raw, output_messages_raw, created_at, updated_at
-    )
-    SELECT
-        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,
-        user_id, project_id, agent_id, agent_version, source, engine, model,
-        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,
-        total_cost_usd, status, error_type, error_message,
-        input_messages_raw, output_messages_raw, now(), now()
-    FROM tmp_model_calls
-    ON CONFLICT(model_call_id) DO UPDATE SET
-        span_id = COALESCE(excluded.span_id, model_calls.span_id),
-        span_name = COALESCE(excluded.span_name, model_calls.span_name),
-        session_id = COALESCE(excluded.session_id, model_calls.session_id),
-        turn_id = COALESCE(excluded.turn_id, model_calls.turn_id),
-        request_id = COALESCE(excluded.request_id, model_calls.request_id),
-        user_id = COALESCE(excluded.user_id, model_calls.user_id),
-        project_id = COALESCE(excluded.project_id, model_calls.project_id),
-        agent_id = COALESCE(excluded.agent_id, model_calls.agent_id),
-        agent_version = COALESCE(excluded.agent_version, model_calls.agent_version),
-        source = COALESCE(excluded.source, model_calls.source),
-        engine = COALESCE(excluded.engine, model_calls.engine),
-        model = COALESCE(excluded.model, model_calls.model),
-        start_ms = COALESCE(excluded.start_ms, model_calls.start_ms),
-        end_ms = COALESCE(excluded.end_ms, model_calls.end_ms),
-        duration_ms = COALESCE(excluded.duration_ms, model_calls.duration_ms),
-        prompt_tokens = COALESCE(excluded.prompt_tokens, model_calls.prompt_tokens),
-        completion_tokens = COALESCE(excluded.completion_tokens, model_calls.completion_tokens),
-        total_tokens = COALESCE(excluded.total_tokens, model_calls.total_tokens),
-        total_cost_usd = COALESCE(excluded.total_cost_usd, model_calls.total_cost_usd),
-        status = COALESCE(excluded.status, model_calls.status),
-        error_type = COALESCE(excluded.error_type, model_calls.error_type),
-        error_message = COALESCE(excluded.error_message, model_calls.error_message),
-        input_messages_raw = COALESCE(excluded.input_messages_raw, model_calls.input_messages_raw),
-        output_messages_raw = COALESCE(excluded.output_messages_raw, model_calls.output_messages_raw),
-        updated_at = now()
-    """)
-
-    # Gold summary table used by existing dashboards
-    conn.execute("""
-    INSERT INTO spans (
-        trace_id, span_id, span_name, model, cost, input_tokens, output_tokens,
-        session_id, user_id, project_id, agent_id, engine, duration_ms, start_ms, created_at
-    )
-    SELECT
-        trace_id,
-        span_id,
-        span_name,
-        model,
-        total_cost_usd,
-        prompt_tokens,
-        completion_tokens,
-        session_id,
-        user_id,
-        project_id,
-        agent_id,
-        engine,
-        duration_ms,
-        start_ms,
-        now()
-    FROM tmp_model_calls
-    ON CONFLICT(trace_id, span_id) DO UPDATE SET
-        span_name = COALESCE(excluded.span_name, spans.span_name),
-        model = COALESCE(excluded.model, spans.model),
-        cost = COALESCE(excluded.cost, spans.cost),
-        input_tokens = COALESCE(excluded.input_tokens, spans.input_tokens),
-        output_tokens = COALESCE(excluded.output_tokens, spans.output_tokens),
-        session_id = COALESCE(excluded.session_id, spans.session_id),
-        user_id = COALESCE(excluded.user_id, spans.user_id),
-        project_id = COALESCE(excluded.project_id, spans.project_id),
-        agent_id = COALESCE(excluded.agent_id, spans.agent_id),
-        engine = COALESCE(excluded.engine, spans.engine),
-        duration_ms = COALESCE(excluded.duration_ms, spans.duration_ms),
-        start_ms = COALESCE(excluded.start_ms, spans.start_ms)
-    """)
-
-    # Turns from model calls
-    conn.execute("""
-    INSERT INTO turns (
-        turn_id, session_id, request_id, turn_index, started_at, ended_at, status,
-        model, provider, prompt_tokens, completion_tokens, total_tokens, cost_usd,
-        trace_id, created_at, updated_at
-    )
-    SELECT
-        turn_id,
-        session_id,
-        request_id,
-        NULL,
-        to_timestamp(start_ms / 1000.0),
-        to_timestamp(end_ms / 1000.0),
-        status,
-        model,
-        engine,
-        prompt_tokens,
-        completion_tokens,
-        total_tokens,
-        total_cost_usd,
-        trace_id,
-        now(),
-        now()
-    FROM tmp_model_calls
-    ON CONFLICT(turn_id) DO UPDATE SET
-        session_id = excluded.session_id,
-        request_id = excluded.request_id,
-        started_at = LEAST(turns.started_at, excluded.started_at),
-        ended_at = GREATEST(turns.ended_at, excluded.ended_at),
-        status = CASE WHEN excluded.status = 'error' THEN 'error' ELSE turns.status END,
-        model = excluded.model,
-        provider = excluded.provider,
-        prompt_tokens = COALESCE(excluded.prompt_tokens, turns.prompt_tokens),
-        completion_tokens = COALESCE(excluded.completion_tokens, turns.completion_tokens),
-        total_tokens = COALESCE(excluded.total_tokens, turns.total_tokens),
-        cost_usd = COALESCE(excluded.cost_usd, turns.cost_usd),
-        trace_id = excluded.trace_id,
-        updated_at = now()
-    """)
-
-    # Parse structured tool calls from raw message payloads.
+    conn.execute('DROP TABLE IF EXISTS tmp_model_calls')
+    conn.execute(f"\n    CREATE TEMP TABLE tmp_model_calls AS\n    WITH raw_spans AS (\n        SELECT unnest(resourceSpans) as rs\n        FROM read_json('{escaped}', format='newline_delimited', ignore_errors=true)\n    ),\n    spans_unnest AS (SELECT unnest(rs.scopeSpans) as ss FROM raw_spans),\n    flat_spans AS (SELECT unnest(ss.spans) as span FROM spans_unnest),\n    parsed AS (\n        SELECT\n            span.traceId as trace_id,\n            span.spanId as span_id,\n            span.name as span_name,\n            span.startTimeUnixNano::BIGINT / 1000000 as start_ms,\n            span.endTimeUnixNano::BIGINT / 1000000 as end_ms,\n            (span.endTimeUnixNano::BIGINT - span.startTimeUnixNano::BIGINT) / 1000000.0 as duration_ms,\n            json_extract(to_json(span), '$.attributes') as attrs_json\n        FROM flat_spans\n    ),\n    attrs AS (\n        SELECT\n            trace_id,\n            span_id,\n            span_name,\n            duration_ms,\n            start_ms,\n            end_ms,\n            json_extract_string(value, '$.key') as attr_key,\n            json_extract_string(value, '$.value.stringValue') as attr_string,\n            json_extract_string(value, '$.value.intValue') as attr_int,\n            json_extract(value, '$.value.doubleValue') as attr_double\n        FROM parsed, json_each(coalesce(attrs_json, '[]'::JSON))\n    ),\n    extracted AS (\n        SELECT\n            trace_id,\n            span_id,\n            span_name,\n            start_ms,\n            end_ms,\n            duration_ms,\n            max(CASE WHEN attr_key IN ('gen_ai.request.model','gen_ai.response.model','model') THEN attr_string END) as model,\n            max(CASE WHEN attr_key = 'gen_ai.cost.total_cost' THEN CAST(attr_double AS DOUBLE) END) as total_cost_usd,\n            max(CASE WHEN attr_key = 'gen_ai.usage.input_tokens' THEN CAST(attr_int AS BIGINT) END) as prompt_tokens,\n            max(CASE WHEN attr_key = 'gen_ai.usage.output_tokens' THEN CAST(attr_int AS BIGINT) END) as completion_tokens,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-session-id'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'session_id'': ''([^'']+)''', 1), '')\n            ) as session_id,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-user-id'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'user_id'': ''([^'']+)''', 1), '')\n            ) as user_id,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-project-id'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'project_id'': ''([^'']+)''', 1), '')\n            ) as project_id,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-id'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_id'': ''([^'']+)''', 1), '')\n            ) as agent_id,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-version'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_version'': ''([^'']+)''', 1), '')\n            ) as agent_version,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-source'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'source'': ''([^'']+)''', 1), '')\n            ) as source,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-engine'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'engine'': ''([^'']+)''', 1), '')\n            ) as engine,\n            COALESCE(\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-request-id'': ''([^'']+)''', 1), ''),\n                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'request_id'': ''([^'']+)''', 1), '')\n            ) as request_id,\n            max(CASE WHEN attr_key IN ('otel.status_code') THEN attr_string END) as otel_status,\n            max(CASE WHEN attr_key IN ('exception.type') THEN attr_string END) as error_type,\n            max(CASE WHEN attr_key IN ('exception.message') THEN attr_string END) as error_message,\n            max(CASE WHEN attr_key IN (\n                'gen_ai.tool.name',\n                'tool.name',\n                'toolName'\n            ) THEN attr_string END) as span_tool_name,\n            max(CASE WHEN attr_key IN (\n                'gen_ai.tool.call.id',\n                'tool.call.id',\n                'tool_call_id',\n                'tool.use.id',\n                'tool_use_id'\n            ) THEN attr_string END) as span_tool_call_id,\n            max(CASE WHEN attr_key IN (\n                'gen_ai.tool.arguments',\n                'tool.arguments',\n                'tool.input',\n                'tool.args'\n            ) THEN attr_string END) as span_tool_arguments_raw,\n            max(CASE WHEN attr_key IN (\n                'gen_ai.tool.result',\n                'tool.result',\n                'tool.output'\n            ) THEN attr_string END) as span_tool_result_raw,\n            max(CASE WHEN attr_key = 'gen_ai.input.messages' THEN attr_string END) as input_messages_raw,\n            max(CASE WHEN attr_key = 'gen_ai.output.messages' THEN attr_string END) as output_messages_raw\n        FROM attrs\n        GROUP BY trace_id, span_id, span_name, duration_ms, start_ms, end_ms\n    ),\n    representative AS (\n        SELECT DISTINCT ON (trace_id)\n            trace_id,\n            span_id,\n            span_name,\n            session_id,\n            request_id,\n            user_id,\n            project_id,\n            agent_id,\n            agent_version,\n            source,\n            engine,\n            model,\n            start_ms,\n            end_ms,\n            duration_ms,\n            COALESCE(prompt_tokens, 0) as prompt_tokens,\n            COALESCE(completion_tokens, 0) as completion_tokens,\n            COALESCE(total_cost_usd, 0.0) as total_cost_usd,\n            error_type,\n            error_message,\n            span_tool_name,\n            span_tool_call_id,\n            span_tool_arguments_raw,\n            span_tool_result_raw,\n            input_messages_raw,\n            output_messages_raw\n        FROM extracted\n        ORDER BY\n            trace_id,\n            CASE WHEN output_messages_raw IS NOT NULL AND output_messages_raw != '' THEN 0 ELSE 1 END,\n            CASE WHEN input_messages_raw IS NOT NULL AND input_messages_raw != '' THEN 0 ELSE 1 END,\n            COALESCE(total_cost_usd, 0) DESC,\n            CASE\n                WHEN span_name = 'raw_gen_ai_request' THEN 0\n                WHEN span_name = 'litellm_request' THEN 1\n                ELSE 2\n            END,\n            start_ms DESC\n    ),\n    merged AS (\n        SELECT\n            e.trace_id,\n            r.span_id,\n            r.span_name,\n            COALESCE(MAX(CASE WHEN e.session_id IS NOT NULL AND e.session_id != '' THEN e.session_id END), r.session_id) AS session_id,\n            COALESCE(MAX(CASE WHEN e.request_id IS NOT NULL AND e.request_id != '' THEN e.request_id END), r.request_id) AS request_id,\n            COALESCE(MAX(CASE WHEN e.user_id IS NOT NULL AND e.user_id != '' THEN e.user_id END), r.user_id) AS user_id,\n            COALESCE(MAX(CASE WHEN e.project_id IS NOT NULL AND e.project_id != '' THEN e.project_id END), r.project_id) AS project_id,\n            COALESCE(MAX(CASE WHEN e.agent_id IS NOT NULL AND e.agent_id != '' THEN e.agent_id END), r.agent_id) AS agent_id,\n            COALESCE(MAX(CASE WHEN e.agent_version IS NOT NULL AND e.agent_version != '' THEN e.agent_version END), r.agent_version) AS agent_version,\n            COALESCE(MAX(CASE WHEN e.source IS NOT NULL AND e.source != '' THEN e.source END), r.source) AS source,\n            COALESCE(MAX(CASE WHEN e.engine IS NOT NULL AND e.engine != '' THEN e.engine END), r.engine) AS engine,\n            COALESCE(MAX(CASE WHEN e.model IS NOT NULL AND e.model != '' THEN e.model END), r.model) AS model,\n            MIN(e.start_ms) AS start_ms,\n            MAX(e.end_ms) AS end_ms,\n            MAX(e.duration_ms) AS duration_ms,\n            COALESCE(MAX(e.prompt_tokens), r.prompt_tokens, 0) AS prompt_tokens,\n            COALESCE(MAX(e.completion_tokens), r.completion_tokens, 0) AS completion_tokens,\n            COALESCE(MAX(e.total_cost_usd), r.total_cost_usd, 0.0) AS total_cost_usd,\n            CASE\n                WHEN SUM(CASE WHEN lower(coalesce(e.otel_status, '')) = 'error' OR e.error_type IS NOT NULL OR e.error_message IS NOT NULL THEN 1 ELSE 0 END) > 0 THEN 'error'\n                ELSE 'ok'\n            END AS status,\n            COALESCE(MAX(CASE WHEN e.error_type IS NOT NULL AND e.error_type != '' THEN e.error_type END), r.error_type) AS error_type,\n            COALESCE(MAX(CASE WHEN e.error_message IS NOT NULL AND e.error_message != '' THEN e.error_message END), r.error_message) AS error_message,\n            COALESCE(MAX(CASE WHEN e.span_tool_name IS NOT NULL AND e.span_tool_name != '' THEN e.span_tool_name END), r.span_tool_name) AS span_tool_name,\n            COALESCE(MAX(CASE WHEN e.span_tool_call_id IS NOT NULL AND e.span_tool_call_id != '' THEN e.span_tool_call_id END), r.span_tool_call_id) AS span_tool_call_id,\n            COALESCE(MAX(CASE WHEN e.span_tool_arguments_raw IS NOT NULL AND e.span_tool_arguments_raw != '' THEN e.span_tool_arguments_raw END), r.span_tool_arguments_raw) AS span_tool_arguments_raw,\n            COALESCE(MAX(CASE WHEN e.span_tool_result_raw IS NOT NULL AND e.span_tool_result_raw != '' THEN e.span_tool_result_raw END), r.span_tool_result_raw) AS span_tool_result_raw,\n            COALESCE(MAX(CASE WHEN e.input_messages_raw IS NOT NULL AND e.input_messages_raw != '' THEN e.input_messages_raw END), r.input_messages_raw) AS input_messages_raw,\n            COALESCE(MAX(CASE WHEN e.output_messages_raw IS NOT NULL AND e.output_messages_raw != '' THEN e.output_messages_raw END), r.output_messages_raw) AS output_messages_raw\n        FROM extracted e\n        JOIN representative r ON r.trace_id = e.trace_id\n        GROUP BY e.trace_id, r.span_id, r.span_name, r.session_id, r.request_id, r.user_id, r.project_id, r.agent_id, r.agent_version, r.source, r.engine, r.model, r.prompt_tokens, r.completion_tokens, r.total_cost_usd, r.error_type, r.error_message, r.span_tool_name, r.span_tool_call_id, r.span_tool_arguments_raw, r.span_tool_result_raw, r.input_messages_raw, r.output_messages_raw\n    )\n    SELECT\n        trace_id as model_call_id,\n        trace_id,\n        span_id,\n        span_name,\n        session_id,\n        CASE\n            WHEN session_id IS NOT NULL AND request_id IS NOT NULL THEN session_id || ':' || request_id\n            ELSE trace_id\n        END as turn_id,\n        request_id,\n        user_id,\n        project_id,\n        agent_id,\n        agent_version,\n        source,\n        engine,\n        model,\n        start_ms,\n        end_ms,\n        duration_ms,\n        prompt_tokens,\n        completion_tokens,\n        COALESCE(prompt_tokens, 0) + COALESCE(completion_tokens, 0) as total_tokens,\n        total_cost_usd,\n        status,\n        error_type,\n        error_message,\n        span_tool_name,\n        span_tool_call_id,\n        span_tool_arguments_raw,\n        span_tool_result_raw,\n        input_messages_raw,\n        output_messages_raw\n    FROM merged\n    ")
+    conn.execute('\n    INSERT INTO model_calls (\n        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,\n        user_id, project_id, agent_id, agent_version, source, engine, model,\n        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,\n        total_cost_usd, status, error_type, error_message,\n        input_messages_raw, output_messages_raw, created_at, updated_at\n    )\n    SELECT\n        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,\n        user_id, project_id, agent_id, agent_version, source, engine, model,\n        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,\n        total_cost_usd, status, error_type, error_message,\n        input_messages_raw, output_messages_raw, now(), now()\n    FROM tmp_model_calls\n    ON CONFLICT(model_call_id) DO UPDATE SET\n        span_id = COALESCE(excluded.span_id, model_calls.span_id),\n        span_name = COALESCE(excluded.span_name, model_calls.span_name),\n        session_id = COALESCE(excluded.session_id, model_calls.session_id),\n        turn_id = COALESCE(excluded.turn_id, model_calls.turn_id),\n        request_id = COALESCE(excluded.request_id, model_calls.request_id),\n        user_id = COALESCE(excluded.user_id, model_calls.user_id),\n        project_id = COALESCE(excluded.project_id, model_calls.project_id),\n        agent_id = COALESCE(excluded.agent_id, model_calls.agent_id),\n        agent_version = COALESCE(excluded.agent_version, model_calls.agent_version),\n        source = COALESCE(excluded.source, model_calls.source),\n        engine = COALESCE(excluded.engine, model_calls.engine),\n        model = COALESCE(excluded.model, model_calls.model),\n        start_ms = COALESCE(excluded.start_ms, model_calls.start_ms),\n        end_ms = COALESCE(excluded.end_ms, model_calls.end_ms),\n        duration_ms = COALESCE(excluded.duration_ms, model_calls.duration_ms),\n        prompt_tokens = COALESCE(excluded.prompt_tokens, model_calls.prompt_tokens),\n        completion_tokens = COALESCE(excluded.completion_tokens, model_calls.completion_tokens),\n        total_tokens = COALESCE(excluded.total_tokens, model_calls.total_tokens),\n        total_cost_usd = COALESCE(excluded.total_cost_usd, model_calls.total_cost_usd),\n        status = COALESCE(excluded.status, model_calls.status),\n        error_type = COALESCE(excluded.error_type, model_calls.error_type),\n        error_message = COALESCE(excluded.error_message, model_calls.error_message),\n        input_messages_raw = COALESCE(excluded.input_messages_raw, model_calls.input_messages_raw),\n        output_messages_raw = COALESCE(excluded.output_messages_raw, model_calls.output_messages_raw),\n        updated_at = now()\n    ')
+    conn.execute('\n    INSERT INTO spans (\n        trace_id, span_id, span_name, model, cost, input_tokens, output_tokens,\n        session_id, user_id, project_id, agent_id, engine, duration_ms, start_ms, created_at\n    )\n    SELECT\n        trace_id,\n        span_id,\n        span_name,\n        model,\n        total_cost_usd,\n        prompt_tokens,\n        completion_tokens,\n        session_id,\n        user_id,\n        project_id,\n        agent_id,\n        engine,\n        duration_ms,\n        start_ms,\n        now()\n    FROM tmp_model_calls\n    ON CONFLICT(trace_id, span_id) DO UPDATE SET\n        span_name = COALESCE(excluded.span_name, spans.span_name),\n        model = COALESCE(excluded.model, spans.model),\n        cost = COALESCE(excluded.cost, spans.cost),\n        input_tokens = COALESCE(excluded.input_tokens, spans.input_tokens),\n        output_tokens = COALESCE(excluded.output_tokens, spans.output_tokens),\n        session_id = COALESCE(excluded.session_id, spans.session_id),\n        user_id = COALESCE(excluded.user_id, spans.user_id),\n        project_id = COALESCE(excluded.project_id, spans.project_id),\n        agent_id = COALESCE(excluded.agent_id, spans.agent_id),\n        engine = COALESCE(excluded.engine, spans.engine),\n        duration_ms = COALESCE(excluded.duration_ms, spans.duration_ms),\n        start_ms = COALESCE(excluded.start_ms, spans.start_ms)\n    ')
+    conn.execute("\n    INSERT INTO turns (\n        turn_id, session_id, request_id, turn_index, started_at, ended_at, status,\n        model, provider, prompt_tokens, completion_tokens, total_tokens, cost_usd,\n        trace_id, created_at, updated_at\n    )\n    SELECT\n        turn_id,\n        session_id,\n        request_id,\n        NULL,\n        to_timestamp(start_ms / 1000.0),\n        to_timestamp(end_ms / 1000.0),\n        status,\n        model,\n        engine,\n        prompt_tokens,\n        completion_tokens,\n        total_tokens,\n        total_cost_usd,\n        trace_id,\n        now(),\n        now()\n    FROM tmp_model_calls\n    ON CONFLICT(turn_id) DO UPDATE SET\n        session_id = excluded.session_id,\n        request_id = excluded.request_id,\n        started_at = LEAST(turns.started_at, excluded.started_at),\n        ended_at = GREATEST(turns.ended_at, excluded.ended_at),\n        status = CASE WHEN excluded.status = 'error' THEN 'error' ELSE turns.status END,\n        model = excluded.model,\n        provider = excluded.provider,\n        prompt_tokens = COALESCE(excluded.prompt_tokens, turns.prompt_tokens),\n        completion_tokens = COALESCE(excluded.completion_tokens, turns.completion_tokens),\n        total_tokens = COALESCE(excluded.total_tokens, turns.total_tokens),\n        cost_usd = COALESCE(excluded.cost_usd, turns.cost_usd),\n        trace_id = excluded.trace_id,\n        updated_at = now()\n    ")
     extract_messages_and_tools(conn)
-
-    # Rebuild session rollups for sessions touched in this file.
-    conn.execute("DROP TABLE IF EXISTS tmp_touched_sessions")
-    conn.execute("""
-    CREATE TEMP TABLE tmp_touched_sessions AS
-    SELECT DISTINCT session_id
-    FROM tmp_model_calls
-    WHERE session_id IS NOT NULL AND session_id != ''
-    """)
-
-    conn.execute("""
-    INSERT INTO sessions (
-        session_id, user_id, project_id, agent_id, agent_version, source, engine,
-        started_at, ended_at, status, total_turns, total_requests,
-        total_cost_usd, total_tokens, created_at, updated_at
-    )
-    SELECT
-        mc.session_id,
-        any_value(mc.user_id) as user_id,
-        any_value(mc.project_id) as project_id,
-        any_value(mc.agent_id) as agent_id,
-        any_value(mc.agent_version) as agent_version,
-        any_value(mc.source) as source,
-        any_value(mc.engine) as engine,
-        to_timestamp(min(mc.start_ms) / 1000.0) as started_at,
-        to_timestamp(max(mc.end_ms) / 1000.0) as ended_at,
-        CASE WHEN sum(CASE WHEN mc.status = 'error' THEN 1 ELSE 0 END) > 0 THEN 'error' ELSE 'ok' END as status,
-        count(distinct mc.turn_id) as total_turns,
-        count(*) as total_requests,
-        sum(coalesce(mc.total_cost_usd, 0.0)) as total_cost_usd,
-        sum(coalesce(mc.total_tokens, 0)) as total_tokens,
-        now(),
-        now()
-    FROM model_calls mc
-    JOIN tmp_touched_sessions t ON t.session_id = mc.session_id
-    GROUP BY mc.session_id
-    ON CONFLICT(session_id) DO UPDATE SET
-        user_id = excluded.user_id,
-        project_id = excluded.project_id,
-        agent_id = excluded.agent_id,
-        agent_version = excluded.agent_version,
-        source = excluded.source,
-        engine = excluded.engine,
-        started_at = excluded.started_at,
-        ended_at = excluded.ended_at,
-        status = excluded.status,
-        total_turns = excluded.total_turns,
-        total_requests = excluded.total_requests,
-        total_cost_usd = excluded.total_cost_usd,
-        total_tokens = excluded.total_tokens,
-        updated_at = now()
-    """)
-
-    # Recompute per-session ordering for turns.
-    conn.execute("""
-    UPDATE turns AS t
-    SET turn_index = ranked.turn_index
-    FROM (
-        SELECT
-            turn_id,
-            row_number() OVER (PARTITION BY session_id ORDER BY started_at, turn_id) - 1 AS turn_index
-        FROM turns
-        WHERE session_id IN (SELECT session_id FROM tmp_touched_sessions)
-    ) ranked
-    WHERE t.turn_id = ranked.turn_id
-    """)
-
+    conn.execute('DROP TABLE IF EXISTS tmp_touched_sessions')
+    conn.execute("\n    CREATE TEMP TABLE tmp_touched_sessions AS\n    SELECT DISTINCT session_id\n    FROM tmp_model_calls\n    WHERE session_id IS NOT NULL AND session_id != ''\n    ")
+    conn.execute("\n    INSERT INTO sessions (\n        session_id, user_id, project_id, agent_id, agent_version, source, engine,\n        started_at, ended_at, status, total_turns, total_requests,\n        total_cost_usd, total_tokens, created_at, updated_at\n    )\n    SELECT\n        mc.session_id,\n        any_value(mc.user_id) as user_id,\n        any_value(mc.project_id) as project_id,\n        any_value(mc.agent_id) as agent_id,\n        any_value(mc.agent_version) as agent_version,\n        any_value(mc.source) as source,\n        any_value(mc.engine) as engine,\n        to_timestamp(min(mc.start_ms) / 1000.0) as started_at,\n        to_timestamp(max(mc.end_ms) / 1000.0) as ended_at,\n        CASE WHEN sum(CASE WHEN mc.status = 'error' THEN 1 ELSE 0 END) > 0 THEN 'error' ELSE 'ok' END as status,\n        count(distinct mc.turn_id) as total_turns,\n        count(*) as total_requests,\n        sum(coalesce(mc.total_cost_usd, 0.0)) as total_cost_usd,\n        sum(coalesce(mc.total_tokens, 0)) as total_tokens,\n        now(),\n        now()\n    FROM model_calls mc\n    JOIN tmp_touched_sessions t ON t.session_id = mc.session_id\n    GROUP BY mc.session_id\n    ON CONFLICT(session_id) DO UPDATE SET\n        user_id = excluded.user_id,\n        project_id = excluded.project_id,\n        agent_id = excluded.agent_id,\n        agent_version = excluded.agent_version,\n        source = excluded.source,\n        engine = excluded.engine,\n        started_at = excluded.started_at,\n        ended_at = excluded.ended_at,\n        status = excluded.status,\n        total_turns = excluded.total_turns,\n        total_requests = excluded.total_requests,\n        total_cost_usd = excluded.total_cost_usd,\n        total_tokens = excluded.total_tokens,\n        updated_at = now()\n    ")
+    conn.execute('\n    UPDATE turns AS t\n    SET turn_index = ranked.turn_index\n    FROM (\n        SELECT\n            turn_id,\n            row_number() OVER (PARTITION BY session_id ORDER BY started_at, turn_id) - 1 AS turn_index\n        FROM turns\n        WHERE session_id IN (SELECT session_id FROM tmp_touched_sessions)\n    ) ranked\n    WHERE t.turn_id = ranked.turn_id\n    ')
 
 def upsert_processed_file(conn, jsonl_file: Path) -> None:
     mtime_ms, size_bytes = get_file_stats(jsonl_file)
-    conn.execute(
-        """
-        INSERT INTO processed_files (file_path, mtime_ms, size_bytes, processed_at)
-        VALUES (?, ?, ?, now())
-        ON CONFLICT(file_path) DO UPDATE SET
-            mtime_ms = excluded.mtime_ms,
-            size_bytes = excluded.size_bytes,
-            processed_at = excluded.processed_at
-        """,
-        [str(jsonl_file), mtime_ms, size_bytes],
-    )
-
+    conn.execute('\n        INSERT INTO processed_files (file_path, mtime_ms, size_bytes, processed_at)\n        VALUES (?, ?, ?, now())\n        ON CONFLICT(file_path) DO UPDATE SET\n            mtime_ms = excluded.mtime_ms,\n            size_bytes = excluded.size_bytes,\n            processed_at = excluded.processed_at\n        ', [str(jsonl_file), mtime_ms, size_bytes])
 
 def main() -> int:
-    """Main entry point."""
     traces_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else None
     db_path = Path(sys.argv[2]) if len(sys.argv) > 2 else None
-
     traces_dir = get_traces_dir(str(traces_dir) if traces_dir else None)
     db_path = get_db_path(traces_dir, str(db_path) if db_path else None)
-
     if not traces_dir.exists():
-        log.error(f"Traces directory not found: {traces_dir}")
-        print(json.dumps({
-            'status': 'error',
-            'indexed': False,
-            'message': f'Traces directory not found: {traces_dir}',
-        }))
+        log.error(f'Traces directory not found: {traces_dir}')
+        print(json.dumps({'status': 'error', 'indexed': False, 'message': f'Traces directory not found: {traces_dir}'}))
         return 1
-
     lock_handle = try_acquire_index_lock(get_lock_file(traces_dir))
     if lock_handle is None:
         log.info('Indexing already in progress')
-        print(json.dumps({
-            'status': 'busy',
-            'indexed': False,
-            'message': 'Indexing already in progress',
-        }))
+        print(json.dumps({'status': 'busy', 'indexed': False, 'message': 'Indexing already in progress'}))
         return 3
-
     try:
         jsonl_files = list(traces_dir.glob('*.jsonl'))
         if not jsonl_files:
-            log.info(f"No JSONL files found in {traces_dir}")
-            print(json.dumps({
-                'status': 'ok',
-                'indexed': False,
-                'message': 'No trace files to index',
-            }))
+            log.info(f'No JSONL files found in {traces_dir}')
+            print(json.dumps({'status': 'ok', 'indexed': False, 'message': 'No trace files to index'}))
             return 2
-
-        log.debug(f"Found {len(jsonl_files)} JSONL files")
-
+        log.debug(f'Found {len(jsonl_files)} JSONL files')
         state_file = get_state_file(traces_dir)
         state = load_index_state(state_file)
-
         db_path.parent.mkdir(parents=True, exist_ok=True)
-
         try:
             import duckdb
         except ImportError:
-            log.error("duckdb module not found. Install with: pip install duckdb")
-            print(json.dumps({
-                'status': 'error',
-                'indexed': False,
-                'message': 'duckdb module not found',
-            }), file=sys.stderr)
+            log.error('duckdb module not found. Install with: pip install duckdb')
+            print(json.dumps({'status': 'error', 'indexed': False, 'message': 'duckdb module not found'}), file=sys.stderr)
             return 1
-
         try:
-            log.debug(f"Connecting to database: {db_path}")
+            log.debug(f'Connecting to database: {db_path}')
             conn = duckdb.connect(str(db_path))
             ensure_schema(conn)
-
             changed_files = get_changed_files(conn, traces_dir)
             if not changed_files:
                 current_mtime = get_latest_jsonl_mtime(traces_dir)
@@ -1339,23 +403,15 @@ def main() -> int:
                 state['duckdbPath'] = str(db_path)
                 state['indexedAt'] = datetime.utcnow().isoformat() + 'Z'
                 save_index_state(state_file, state)
-
-                log.info("Index is up-to-date; no changed files")
-                print(json.dumps({
-                    'status': 'ok',
-                    'indexed': True,
-                    'message': 'Already indexed',
-                }))
+                log.info('Index is up-to-date; no changed files')
+                print(json.dumps({'status': 'ok', 'indexed': True, 'message': 'Already indexed'}))
                 conn.close()
                 return 2
-
-            log.info(f"Incremental indexing {len(changed_files)} changed file(s)")
-
+            log.info(f'Incremental indexing {len(changed_files)} changed file(s)')
             for jsonl_file in changed_files:
-                log.debug(f"Indexing file: {jsonl_file}")
+                log.debug(f'Indexing file: {jsonl_file}')
                 ingest_single_jsonl_file(conn, jsonl_file)
                 upsert_processed_file(conn, jsonl_file)
-
             current_mtime = get_latest_jsonl_mtime(traces_dir)
             state['version'] = 2
             state['indexed'] = True
@@ -1363,32 +419,18 @@ def main() -> int:
             state['duckdbPath'] = str(db_path)
             state['indexedAt'] = datetime.utcnow().isoformat() + 'Z'
             save_index_state(state_file, state)
-
-            count_result = conn.execute("SELECT COUNT(*) FROM model_calls").fetchall()
+            count_result = conn.execute('SELECT COUNT(*) FROM model_calls').fetchall()
             total_records = count_result[0][0] if count_result else 0
-            log.debug(f"Ingestion complete. Total model_calls: {total_records}")
-
+            log.debug(f'Ingestion complete. Total model_calls: {total_records}')
             conn.close()
-
-            log.info(f"Indexing complete: {db_path}")
-            print(json.dumps({
-                'status': 'ok',
-                'indexed': True,
-                'message': f'Indexed {len(changed_files)} changed file(s) at {state["indexedAt"]}',
-            }))
+            log.info(f'Indexing complete: {db_path}')
+            print(json.dumps({'status': 'ok', 'indexed': True, 'message': f'Indexed {len(changed_files)} changed file(s) at {state['indexedAt']}'}))
             return 0
-
         except Exception as e:
-            log.error(f"Indexing failed: {e}")
-            print(json.dumps({
-                'status': 'error',
-                'indexed': False,
-                'message': str(e),
-            }), file=sys.stderr)
+            log.error(f'Indexing failed: {e}')
+            print(json.dumps({'status': 'error', 'indexed': False, 'message': str(e)}), file=sys.stderr)
             return 1
     finally:
         release_index_lock(lock_handle)
-
-
 if __name__ == '__main__':
     sys.exit(main())