@agenthippo-ai/agenthippo 0.1.0

package/out/scripts/analytics/index-traces.py

@@ -0,0 +1,1394 @@
+#!/usr/bin/env python3
+# Copyright (c) AgentHippo.ai. All rights reserved.
+"""
+Incremental JSONL → DuckDB indexing worker.
+
+Converts OpenTelemetry JSONL trace files to indexed DuckDB database for fast analytics queries.
+
+Usage:
+  python3 index-traces.py <traces_dir>
+  python3 index-traces.py <traces_dir> <output_db_path>
+
+Environment variables:
+  AGENTIDE_TRACES_DIR - Path to traces directory (can also be passed as arg)
+  AGENTIDE_INDEX_DB_PATH - Path to output .duckdb file
+
+Exit codes:
+  0 = success (indexing completed)
+  1 = error (indexing failed)
+  2 = already indexed (no changes needed)
+  3 = indexing already in progress
+"""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+import json
+import logging
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+# Setup logging
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(levelname)s: %(message)s',
+    stream=sys.stderr,
+)
+log = logging.getLogger(__name__)
+
+
+def get_traces_dir(arg_traces_dir: str | None = None) -> Path:
+    """Get traces directory from arg, env var, or default."""
+    if arg_traces_dir:
+        return Path(arg_traces_dir)
+
+    env_traces_dir = os.environ.get('AGENTIDE_TRACES_DIR', '').strip()
+    if env_traces_dir:
+        return Path(env_traces_dir)
+
+    # Default: ~/.agent-hippo/analytics/traces
+    agent_home = Path.home() / '.agent-hippo'
+    return agent_home / 'analytics' / 'traces'
+
+
+def get_db_path(traces_dir: Path, arg_db_path: str | None = None) -> Path:
+    """Get database path from arg, env var, or default (traces_dir/analytics.duckdb)."""
+    if arg_db_path:
+        return Path(arg_db_path)
+
+    env_db_path = os.environ.get('AGENTIDE_INDEX_DB_PATH', '').strip()
+    if env_db_path:
+        return Path(env_db_path)
+
+    return traces_dir / 'analytics.duckdb'
+
+
+def get_state_file(traces_dir: Path) -> Path:
+    """Get indexing state file path."""
+    return traces_dir / '.duckdb-index.json'
+
+
+def get_lock_file(traces_dir: Path) -> Path:
+    """Get process lock file path."""
+    return traces_dir / '.indexing.lock'
+
+
+def try_acquire_index_lock(lock_file: Path):
+    """
+    Try to acquire a cross-process exclusive lock.
+    Returns an open file handle when acquired, otherwise None.
+    """
+    lock_file.parent.mkdir(parents=True, exist_ok=True)
+    handle = lock_file.open('a+b')
+
+    # Ensure the file has at least one byte for Windows byte-range locking.
+    handle.seek(0, os.SEEK_END)
+    if handle.tell() == 0:
+        handle.write(b'0')
+        handle.flush()
+
+    try:
+        if os.name == 'nt':
+            import msvcrt
+            handle.seek(0)
+            msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+        else:
+            import fcntl
+            fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        handle.close()
+        return None
+
+    # Record owner PID for diagnostics.
+    handle.seek(0)
+    handle.truncate(0)
+    handle.write(f"{os.getpid()}\n".encode('utf-8'))
+    handle.flush()
+
+    return handle
+
+
+def release_index_lock(handle) -> None:
+    """Release the process lock (best effort)."""
+    if handle is None:
+        return
+
+    try:
+        if os.name == 'nt':
+            import msvcrt
+            handle.seek(0)
+            msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+        else:
+            import fcntl
+            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+    except OSError:
+        pass
+    finally:
+        handle.close()
+
+
+def load_index_state(state_file: Path) -> dict[str, Any]:
+    """Load indexing state from disk."""
+    try:
+        if state_file.exists():
+            content = state_file.read_text('utf-8')
+            return json.loads(content)
+    except Exception as e:
+        log.debug(f"Failed to load state file: {e}")
+
+    return {
+        'version': 2,
+        'indexed': False,
+        'lastMtime': 0,
+        'duckdbPath': '',
+    }
+
+
+def save_index_state(state_file: Path, state: dict[str, Any]) -> None:
+    """Save indexing state to disk."""
+    try:
+        state_file.write_text(json.dumps(state, indent=2), 'utf-8')
+    except Exception as e:
+        log.warning(f"Failed to save state file: {e}")
+
+
+def get_latest_jsonl_mtime(traces_dir: Path) -> int:
+    """Get maximum mtime of all JSONL files in traces_dir."""
+    try:
+        max_mtime = 0
+        for jsonl_file in traces_dir.glob('*.jsonl'):
+            try:
+                mtime_ms = int(jsonl_file.stat().st_mtime * 1000)
+                max_mtime = max(max_mtime, mtime_ms)
+            except OSError:
+                pass
+        return max_mtime
+    except Exception as e:
+        log.error(f"Failed to get JSONL mtimes: {e}")
+        return 0
+
+
+def ensure_schema(conn) -> None:
+    """Create normalized LLM-agent analytics tables, marts, and indexes."""
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS model_calls (
+        model_call_id VARCHAR PRIMARY KEY,
+        trace_id VARCHAR,
+        span_id VARCHAR,
+        span_name VARCHAR,
+        session_id VARCHAR,
+        turn_id VARCHAR,
+        request_id VARCHAR,
+        user_id VARCHAR,
+        project_id VARCHAR,
+        agent_id VARCHAR,
+        agent_version VARCHAR,
+        source VARCHAR,
+        engine VARCHAR,
+        model VARCHAR,
+        start_ms BIGINT,
+        end_ms BIGINT,
+        duration_ms DOUBLE,
+        prompt_tokens BIGINT,
+        completion_tokens BIGINT,
+        total_tokens BIGINT,
+        total_cost_usd DOUBLE,
+        status VARCHAR,
+        error_type VARCHAR,
+        error_message VARCHAR,
+        input_messages_raw VARCHAR,
+        output_messages_raw VARCHAR,
+        created_at TIMESTAMP,
+        updated_at TIMESTAMP
+    )
+    """)
+
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS sessions (
+        session_id VARCHAR PRIMARY KEY,
+        user_id VARCHAR,
+        project_id VARCHAR,
+        agent_id VARCHAR,
+        agent_version VARCHAR,
+        source VARCHAR,
+        engine VARCHAR,
+        started_at TIMESTAMP,
+        ended_at TIMESTAMP,
+        status VARCHAR,
+        total_turns BIGINT,
+        total_requests BIGINT,
+        total_cost_usd DOUBLE,
+        total_tokens BIGINT,
+        created_at TIMESTAMP,
+        updated_at TIMESTAMP
+    )
+    """)
+
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS turns (
+        turn_id VARCHAR PRIMARY KEY,
+        session_id VARCHAR,
+        request_id VARCHAR,
+        turn_index BIGINT,
+        started_at TIMESTAMP,
+        ended_at TIMESTAMP,
+        status VARCHAR,
+        model VARCHAR,
+        provider VARCHAR,
+        prompt_tokens BIGINT,
+        completion_tokens BIGINT,
+        total_tokens BIGINT,
+        cost_usd DOUBLE,
+        trace_id VARCHAR,
+        created_at TIMESTAMP,
+        updated_at TIMESTAMP
+    )
+    """)
+
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS tool_calls (
+        tool_call_id VARCHAR PRIMARY KEY,
+        session_id VARCHAR,
+        turn_id VARCHAR,
+        request_id VARCHAR,
+        model_call_id VARCHAR,
+        trace_id VARCHAR,
+        span_id VARCHAR,
+        tool_name VARCHAR,
+        arguments_json JSON,
+        arguments_text VARCHAR,
+        call_ts TIMESTAMP,
+        result_json JSON,
+        result_text VARCHAR,
+        status VARCHAR,
+        duration_ms DOUBLE,
+        error_type VARCHAR,
+        error_message VARCHAR,
+        created_at TIMESTAMP,
+        updated_at TIMESTAMP
+    )
+    """)
+
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS artifacts (
+        artifact_id VARCHAR PRIMARY KEY,
+        session_id VARCHAR,
+        turn_id VARCHAR,
+        trace_id VARCHAR,
+        kind VARCHAR,
+        mime_type VARCHAR,
+        storage_uri VARCHAR,
+        sha256 VARCHAR,
+        byte_size BIGINT,
+        created_at TIMESTAMP
+    )
+    """)
+
+    # Gold compatibility/fast dashboard table
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS spans (
+        trace_id VARCHAR,
+        span_id VARCHAR,
+        span_name VARCHAR,
+        model VARCHAR,
+        cost DOUBLE,
+        input_tokens BIGINT,
+        output_tokens BIGINT,
+        session_id VARCHAR,
+        user_id VARCHAR,
+        project_id VARCHAR,
+        agent_id VARCHAR,
+        engine VARCHAR,
+        duration_ms DOUBLE,
+        start_ms BIGINT,
+        created_at TIMESTAMP,
+        PRIMARY KEY(trace_id, span_id)
+    )
+    """)
+    try:
+        conn.execute("ALTER TABLE spans ADD COLUMN IF NOT EXISTS engine VARCHAR")
+    except Exception as e:
+        log.debug(f"Spans engine column (may already exist): {e}")
+
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS processed_files (
+        file_path VARCHAR PRIMARY KEY,
+        mtime_ms BIGINT NOT NULL,
+        size_bytes BIGINT NOT NULL,
+        processed_at TIMESTAMP NOT NULL
+    )
+    """)
+
+    conn.execute("""
+    CREATE OR REPLACE VIEW agent_kpis_daily AS
+    SELECT
+        strftime(to_timestamp(start_ms / 1000.0), '%Y-%m-%d') AS day,
+        coalesce(agent_id, '(no agent)') AS agent_id,
+        coalesce(agent_version, '(unknown)') AS agent_version,
+        coalesce(model, '(unknown)') AS model,
+        count(*) AS requests,
+        count(distinct session_id) AS sessions,
+        count(distinct turn_id) AS turns,
+        round(sum(coalesce(total_cost_usd, 0.0)), 6) AS total_cost_usd,
+        sum(coalesce(total_tokens, 0)) AS total_tokens,
+        round(avg(duration_ms), 2) AS avg_latency_ms,
+        round(percentile_cont(0.50) within group (order by duration_ms), 2) AS p50_latency_ms,
+        round(percentile_cont(0.95) within group (order by duration_ms), 2) AS p95_latency_ms,
+        round(100.0 * sum(case when status = 'error' then 1 else 0 end) / nullif(count(*), 0), 2) AS error_rate_pct
+    FROM model_calls
+    GROUP BY 1, 2, 3, 4
+    """)
+
+    # Common indexes for dashboard and skill queries.
+    for stmt in [
+        "CREATE INDEX IF NOT EXISTS idx_model_calls_session_id ON model_calls(session_id)",
+        "CREATE INDEX IF NOT EXISTS idx_model_calls_start_ms ON model_calls(start_ms)",
+        "CREATE INDEX IF NOT EXISTS idx_model_calls_agent_id ON model_calls(agent_id)",
+        "CREATE INDEX IF NOT EXISTS idx_model_calls_project_id ON model_calls(project_id)",
+        "CREATE INDEX IF NOT EXISTS idx_tool_calls_session ON tool_calls(session_id)",
+        "CREATE INDEX IF NOT EXISTS idx_spans_session ON spans(session_id)",
+        "CREATE INDEX IF NOT EXISTS idx_spans_start_ms ON spans(start_ms)",
+        "CREATE INDEX IF NOT EXISTS idx_spans_model ON spans(model)",
+        "CREATE INDEX IF NOT EXISTS idx_spans_agent_id ON spans(agent_id)",
+        "CREATE INDEX IF NOT EXISTS idx_spans_project_id ON spans(project_id)",
+    ]:
+        try:
+            conn.execute(stmt)
+        except Exception as e:
+            log.debug(f"Index creation hint (may already exist): {e}")
+
+
+def get_file_stats(jsonl_file: Path) -> tuple[int, int]:
+    stats = jsonl_file.stat()
+    return int(stats.st_mtime * 1000), int(stats.st_size)
+
+
+def get_processed_file_map(conn) -> dict[str, tuple[int, int]]:
+    rows = conn.execute(
+        "SELECT file_path, mtime_ms, size_bytes FROM processed_files"
+    ).fetchall()
+    return {row[0]: (int(row[1]), int(row[2])) for row in rows}
+
+
+def get_changed_files(conn, traces_dir: Path) -> list[Path]:
+    processed = get_processed_file_map(conn)
+    changed: list[Path] = []
+
+    for jsonl_file in sorted(traces_dir.glob('*.jsonl')):
+        try:
+            mtime_ms, size_bytes = get_file_stats(jsonl_file)
+        except OSError:
+            continue
+
+        key = str(jsonl_file)
+        previous = processed.get(key)
+        if previous is None or previous[0] != mtime_ms or previous[1] != size_bytes:
+            changed.append(jsonl_file)
+
+    return changed
+
+
+def _safe_json_parse(raw: Any) -> Any:
+    if raw is None:
+        return None
+    if isinstance(raw, (dict, list)):
+        return raw
+
+    text = str(raw).strip()
+    if text == '':
+        return None
+
+    # try strict JSON first
+    try:
+        parsed = json.loads(text)
+    except Exception:
+        parsed = None
+
+    # fallback to python-literal-like payloads (single-quoted dict/list)
+    if parsed is None:
+        try:
+            parsed = ast.literal_eval(text)
+        except Exception:
+            parsed = None
+
+    if isinstance(parsed, str) and parsed != text:
+        return _safe_json_parse(parsed)
+
+    return parsed
+
+
+def _to_json_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    try:
+        return json.dumps(value, ensure_ascii=False)
+    except Exception:
+        return None
+
+
+def _extract_text_content(content: Any) -> str:
+    if content is None:
+        return ''
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for item in content:
+            if isinstance(item, str):
+                parts.append(item)
+                continue
+            if isinstance(item, dict):
+                item_type = str(item.get('type', '')).lower()
+                if item_type in {'thinking', 'signature'}:
+                    continue
+                text = item.get('text')
+                if isinstance(text, str) and text:
+                    parts.append(text)
+                elif item.get('content') is not None:
+                    parts.append(_extract_text_content(item.get('content')))
+        return '\n'.join([p for p in parts if p])
+    if isinstance(content, dict):
+        if isinstance(content.get('text'), str):
+            return content.get('text', '')
+        if content.get('content') is not None:
+            return _extract_text_content(content.get('content'))
+        if content.get('input') is not None:
+            return _extract_text_content(content.get('input'))
+    return str(content)
+
+
+def _normalize_messages(payload: Any, default_role: str) -> list[dict[str, Any]]:
+    parsed = _safe_json_parse(payload)
+
+    if parsed is None:
+        return []
+
+    if isinstance(parsed, dict):
+        if isinstance(parsed.get('messages'), list):
+            parsed = parsed['messages']
+        elif isinstance(parsed.get('choices'), list):
+            out: list[dict[str, Any]] = []
+            for choice in parsed.get('choices', []):
+                if isinstance(choice, dict):
+                    msg = choice.get('message')
+                    if isinstance(msg, dict):
+                        out.append(msg)
+            return out
+        else:
+            parsed = [parsed]
+
+    if not isinstance(parsed, list):
+        return [{"role": default_role, "content": str(parsed)}]
+
+    out: list[dict[str, Any]] = []
+    for entry in parsed:
+        if isinstance(entry, dict):
+            out.append(entry)
+        else:
+            out.append({"role": default_role, "content": str(entry)})
+    return out
+
+
+def _tool_call_id(session_id: str | None, trace_id: str, call_id: str | None, idx: int) -> str:
+    if call_id:
+        return call_id
+    seed = f"{session_id or ''}|{trace_id}|tool|{idx}"
+    return hashlib.sha1(seed.encode('utf-8')).hexdigest()
+
+
+def extract_messages_and_tools(conn) -> None:
+    """
+    Parse tool-call payloads from current temp batch (tmp_model_calls)
+    and upsert into tool_calls.
+    """
+    rows = conn.execute(
+        """
+        SELECT
+            model_call_id,
+            trace_id,
+            span_id,
+            session_id,
+            turn_id,
+            request_id,
+            start_ms,
+            end_ms,
+            status,
+            error_type,
+            error_message,
+            input_messages_raw,
+            output_messages_raw,
+            span_tool_name,
+            span_tool_call_id,
+            span_tool_arguments_raw,
+            span_tool_result_raw
+        FROM tmp_model_calls
+        """
+    ).fetchall()
+
+    tool_results_by_id: dict[str, dict[str, Any]] = {}
+
+    for row in rows:
+        (
+            model_call_id,
+            trace_id,
+            span_id,
+            session_id,
+            turn_id,
+            request_id,
+            start_ms,
+            end_ms,
+            call_status,
+            call_error_type,
+            call_error_message,
+            input_raw,
+            output_raw,
+            span_tool_name,
+            span_tool_call_id,
+            span_tool_arguments_raw,
+            span_tool_result_raw,
+        ) = row
+
+        tool_call_inserted = False
+        tool_base_ts = int(end_ms) if end_ms is not None else int(start_ms) if start_ms is not None else 0
+
+        if span_tool_name:
+            fallback_call_id = span_tool_call_id
+            if fallback_call_id is not None:
+                fallback_call_id = str(fallback_call_id)
+            fallback_tool_call_id = _tool_call_id(session_id, str(trace_id), fallback_call_id, 0)
+            fallback_status = 'error' if str(call_status or '').lower() == 'error' else 'called'
+            fallback_result_text = _extract_text_content(span_tool_result_raw)
+            fallback_result_json = _to_json_text(_safe_json_parse(span_tool_result_raw) if span_tool_result_raw else None)
+            if fallback_result_json is None and span_tool_result_raw:
+                fallback_result_json = _to_json_text(span_tool_result_raw)
+
+            conn.execute(
+                """
+                INSERT INTO tool_calls (
+                    tool_call_id, session_id, turn_id, request_id, model_call_id,
+                    trace_id, span_id, tool_name, arguments_json, arguments_text,
+                    call_ts, result_json, result_text, status, duration_ms,
+                    error_type, error_message, created_at, updated_at
+                )
+                VALUES (
+                    ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,
+                    to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,
+                    ?, ?, now(), now()
+                )
+                ON CONFLICT(tool_call_id) DO UPDATE SET
+                    result_json = COALESCE(excluded.result_json, tool_calls.result_json),
+                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),
+                    status = CASE
+                        WHEN excluded.status = 'error' THEN 'error'
+                        WHEN excluded.status = 'ok' THEN 'ok'
+                        ELSE tool_calls.status
+                    END,
+                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),
+                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),
+                    updated_at = now()
+                """,
+                [
+                    fallback_tool_call_id,
+                    session_id,
+                    turn_id,
+                    request_id,
+                    model_call_id,
+                    trace_id,
+                    span_id,
+                    str(span_tool_name),
+                    _to_json_text(_safe_json_parse(span_tool_arguments_raw) if span_tool_arguments_raw else None)
+                    or _to_json_text(span_tool_arguments_raw),
+                    _extract_text_content(span_tool_arguments_raw),
+                    tool_base_ts,
+                    fallback_result_json,
+                    fallback_result_text,
+                    fallback_status,
+                    call_error_type,
+                    call_error_message,
+                ],
+            )
+            tool_call_inserted = True
+
+        if str(call_status or '').lower() == 'error' and not tool_call_inserted and call_error_message:
+            synthetic_tool_call_id = _tool_call_id(session_id, str(trace_id), None, 0)
+            conn.execute(
+                """
+                INSERT INTO tool_calls (
+                    tool_call_id, session_id, turn_id, request_id, model_call_id,
+                    trace_id, span_id, tool_name, arguments_json, arguments_text,
+                    call_ts, result_json, result_text, status, duration_ms,
+                    error_type, error_message, created_at, updated_at
+                )
+                VALUES (
+                    ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL,
+                    to_timestamp(? / 1000.0), NULL, ?, 'error', NULL,
+                    ?, ?, now(), now()
+                )
+                ON CONFLICT(tool_call_id) DO UPDATE SET
+                    status = 'error',
+                    result_text = COALESCE(excluded.result_text, tool_calls.result_text),
+                    error_type = COALESCE(excluded.error_type, tool_calls.error_type),
+                    error_message = COALESCE(excluded.error_message, tool_calls.error_message),
+                    updated_at = now()
+                """,
+                [
+                    synthetic_tool_call_id,
+                    session_id,
+                    turn_id,
+                    request_id,
+                    model_call_id,
+                    trace_id,
+                    span_id,
+                    '(tool-error)',
+                    tool_base_ts,
+                    call_error_message,
+                    call_error_type,
+                    call_error_message,
+                ],
+            )
+
+        if span_tool_call_id:
+            tool_results_by_id[str(span_tool_call_id)] = {
+                'result_json': _to_json_text(_safe_json_parse(span_tool_result_raw) if span_tool_result_raw else None),
+                'result_text': _extract_text_content(span_tool_result_raw),
+                'status': 'error' if str(call_status or '').lower() == 'error' else 'ok',
+                'updated_at_ms': tool_base_ts,
+            }
+            if tool_results_by_id[str(span_tool_call_id)]['result_json'] is None and span_tool_result_raw:
+                tool_results_by_id[str(span_tool_call_id)]['result_json'] = _to_json_text(span_tool_result_raw)
+
+        input_messages = _normalize_messages(input_raw, 'user')
+        output_messages = _normalize_messages(output_raw, 'assistant')
+
+        # tool_result blocks may appear in input messages and complete tool calls emitted in output
+        for i, msg in enumerate(input_messages):
+            content = msg.get('content', msg.get('parts', msg.get('text', '')))
+            if isinstance(content, list):
+                for j, block in enumerate(content):
+                    if not isinstance(block, dict):
+                        continue
+                    block_type = str(block.get('type', '')).lower()
+                    if block_type != 'tool_result':
+                        continue
+                    call_id = block.get('tool_use_id') or block.get('call_id') or block.get('id')
+                    if not call_id:
+                        continue
+                    tool_results_by_id[str(call_id)] = {
+                        'result_json': _to_json_text(block.get('content', block.get('result'))),
+                        'result_text': _extract_text_content(block.get('content', block.get('result'))),
+                        'status': 'ok',
+                        'updated_at_ms': int(start_ms) + j if start_ms is not None else 0,
+                    }
+
+        # Output-side tool calls
+        for i, msg in enumerate(output_messages):
+            content = msg.get('content', msg.get('parts', msg.get('text', '')))
+            output_ts = int(end_ms) if end_ms is not None else int(start_ms) if start_ms is not None else 0
+
+            # Normalize tool blocks from multiple provider formats.
+            tool_blocks: list[dict[str, Any]] = []
+            if isinstance(content, list):
+                tool_blocks.extend([b for b in content if isinstance(b, dict)])
+
+            direct_tool_calls = msg.get('tool_calls')
+            if isinstance(direct_tool_calls, list):
+                for block in direct_tool_calls:
+                    if isinstance(block, dict):
+                        normalized = dict(block)
+                        normalized.setdefault('type', 'tool_call')
+                        tool_blocks.append(normalized)
+
+            function_call = msg.get('function_call')
+            if isinstance(function_call, dict):
+                tool_blocks.append({'type': 'function_call', 'function': function_call})
+
+            if not tool_blocks:
+                continue
+
+            for j, block in enumerate(tool_blocks):
+                if not isinstance(block, dict):
+                    continue
+
+                block_type = str(block.get('type', '')).lower()
+                if block_type not in {'tool_use', 'tool_call', 'function_call', 'server_tool_use', 'mcp_tool_use'}:
+                    continue
+
+                if block_type in {'server_tool_use', 'mcp_tool_use'} and not block.get('name'):
+                    # Normalize Anthropic streaming variants that still represent tool invocation.
+                    block['name'] = block.get('tool_name') or block.get('toolName')
+                    block.setdefault('input', block.get('arguments'))
+
+                if block_type == 'tool_call' and not block.get('function') and isinstance(block.get('name'), str):
+                    block['function'] = {
+                        'name': block.get('name'),
+                        'arguments': block.get('arguments') or block.get('input'),
+                    }
+
+                if block_type == 'function_call' and not block.get('function') and (block.get('name') or block.get('arguments')):
+                    block['function'] = {
+                        'name': block.get('name'),
+                        'arguments': block.get('arguments'),
+                    }
+
+                if block_type == 'tool_use' and not block.get('name'):
+                    block['name'] = block.get('tool_name') or block.get('toolName')
+
+                if block_type == 'tool_use' and block.get('input') is None and block.get('arguments') is not None:
+                    block['input'] = block.get('arguments')
+
+                if block_type in {'tool_call', 'function_call'} and block.get('function') is None and block.get('name'):
+                    block['function'] = {
+                        'name': block.get('name'),
+                        'arguments': block.get('arguments') or block.get('input'),
+                    }
+
+                if block_type == 'function_call' and block.get('function') is None:
+                    continue
+
+                if block_type == 'tool_call' and block.get('function') is None and block.get('name') is None:
+                    continue
+
+                if block_type in {'server_tool_use', 'mcp_tool_use'} and not block.get('name'):
+                    continue
+
+                if block_type == 'tool_use' and not block.get('name'):
+                    continue
+
+                if block_type == 'tool_call' and block.get('function') is None and not block.get('name'):
+                    continue
+
+                if block_type == 'function_call' and block.get('function') is None:
+                    continue
+
+                # Harmonize final type for downstream extraction.
+                if block_type in {'server_tool_use', 'mcp_tool_use'}:
+                    block_type = 'tool_use'
+
+                block['type'] = block_type
+
+
+                call_id = block.get('id') or block.get('tool_call_id') or block.get('call_id')
+                tool_name = (
+                    block.get('name')
+                    or (block.get('function', {}) or {}).get('name')
+                    or 'unknown_tool'
+                )
+
+                args = (
+                    block.get('input')
+                    if block.get('input') is not None
+                    else (block.get('function', {}) or {}).get('arguments')
+                )
+                args_json = _to_json_text(args)
+                args_text = _extract_text_content(args)
+
+                tool_call_id = _tool_call_id(session_id, str(trace_id), str(call_id) if call_id else None, j)
+                tool_result = tool_results_by_id.get(str(call_id)) if call_id else None
+
+                result_json = tool_result['result_json'] if tool_result else None
+                result_text = tool_result['result_text'] if tool_result else None
+                status = tool_result['status'] if tool_result else 'called'
+
+                conn.execute(
+                    """
+                    INSERT INTO tool_calls (
+                        tool_call_id, session_id, turn_id, request_id, model_call_id,
+                        trace_id, span_id, tool_name, arguments_json, arguments_text,
+                        call_ts, result_json, result_text, status, duration_ms,
+                        error_type, error_message, created_at, updated_at
+                    )
+                    VALUES (
+                        ?, ?, ?, ?, ?, ?, ?, ?, ?::JSON, ?,
+                        to_timestamp(? / 1000.0), ?::JSON, ?, ?, NULL,
+                        NULL, NULL, now(), now()
+                    )
+                    ON CONFLICT(tool_call_id) DO UPDATE SET
+                        result_json = COALESCE(excluded.result_json, tool_calls.result_json),
+                        result_text = COALESCE(excluded.result_text, tool_calls.result_text),
+                        status = CASE
+                            WHEN excluded.status = 'ok' THEN 'ok'
+                            ELSE tool_calls.status
+                        END,
+                        updated_at = now()
+                    """,
+                    [
+                        tool_call_id,
+                        session_id,
+                        turn_id,
+                        request_id,
+                        model_call_id,
+                        trace_id,
+                        span_id,
+                        str(tool_name),
+                        args_json,
+                        args_text,
+                        output_ts + j,
+                        result_json,
+                        result_text,
+                        status,
+                    ],
+                )
+
+
+def ingest_single_jsonl_file(conn, jsonl_file: Path) -> None:
+    """Ingest one JSONL file into Silver and Gold analytics tables."""
+    escaped = str(jsonl_file).replace("'", "''")
+
+    conn.execute("DROP TABLE IF EXISTS tmp_model_calls")
+    conn.execute(f"""
+    CREATE TEMP TABLE tmp_model_calls AS
+    WITH raw_spans AS (
+        SELECT unnest(resourceSpans) as rs
+        FROM read_json('{escaped}', format='newline_delimited', ignore_errors=true)
+    ),
+    spans_unnest AS (SELECT unnest(rs.scopeSpans) as ss FROM raw_spans),
+    flat_spans AS (SELECT unnest(ss.spans) as span FROM spans_unnest),
+    parsed AS (
+        SELECT
+            span.traceId as trace_id,
+            span.spanId as span_id,
+            span.name as span_name,
+            span.startTimeUnixNano::BIGINT / 1000000 as start_ms,
+            span.endTimeUnixNano::BIGINT / 1000000 as end_ms,
+            (span.endTimeUnixNano::BIGINT - span.startTimeUnixNano::BIGINT) / 1000000.0 as duration_ms,
+            json_extract(to_json(span), '$.attributes') as attrs_json
+        FROM flat_spans
+    ),
+    attrs AS (
+        SELECT
+            trace_id,
+            span_id,
+            span_name,
+            duration_ms,
+            start_ms,
+            end_ms,
+            json_extract_string(value, '$.key') as attr_key,
+            json_extract_string(value, '$.value.stringValue') as attr_string,
+            json_extract_string(value, '$.value.intValue') as attr_int,
+            json_extract(value, '$.value.doubleValue') as attr_double
+        FROM parsed, json_each(coalesce(attrs_json, '[]'::JSON))
+    ),
+    extracted AS (
+        SELECT
+            trace_id,
+            span_id,
+            span_name,
+            start_ms,
+            end_ms,
+            duration_ms,
+            max(CASE WHEN attr_key IN ('gen_ai.request.model','gen_ai.response.model','model') THEN attr_string END) as model,
+            max(CASE WHEN attr_key = 'gen_ai.cost.total_cost' THEN CAST(attr_double AS DOUBLE) END) as total_cost_usd,
+            max(CASE WHEN attr_key = 'gen_ai.usage.input_tokens' THEN CAST(attr_int AS BIGINT) END) as prompt_tokens,
+            max(CASE WHEN attr_key = 'gen_ai.usage.output_tokens' THEN CAST(attr_int AS BIGINT) END) as completion_tokens,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-session-id'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'session_id'': ''([^'']+)''', 1), '')
+            ) as session_id,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-user-id'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'user_id'': ''([^'']+)''', 1), '')
+            ) as user_id,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-project-id'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'project_id'': ''([^'']+)''', 1), '')
+            ) as project_id,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-id'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_id'': ''([^'']+)''', 1), '')
+            ) as agent_id,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-agent-version'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'agent_version'': ''([^'']+)''', 1), '')
+            ) as agent_version,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-source'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'source'': ''([^'']+)''', 1), '')
+            ) as source,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-engine'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'engine'': ''([^'']+)''', 1), '')
+            ) as engine,
+            COALESCE(
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END), 'x-litellm-metadata-request-id'': ''([^'']+)''', 1), ''),
+                NULLIF(regexp_extract(max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END), 'request_id'': ''([^'']+)''', 1), '')
+            ) as request_id,
+            max(CASE WHEN attr_key IN ('otel.status_code') THEN attr_string END) as otel_status,
+            max(CASE WHEN attr_key IN ('exception.type') THEN attr_string END) as error_type,
+            max(CASE WHEN attr_key IN ('exception.message') THEN attr_string END) as error_message,
+            max(CASE WHEN attr_key IN (
+                'gen_ai.tool.name',
+                'tool.name',
+                'toolName'
+            ) THEN attr_string END) as span_tool_name,
+            max(CASE WHEN attr_key IN (
+                'gen_ai.tool.call.id',
+                'tool.call.id',
+                'tool_call_id',
+                'tool.use.id',
+                'tool_use_id'
+            ) THEN attr_string END) as span_tool_call_id,
+            max(CASE WHEN attr_key IN (
+                'gen_ai.tool.arguments',
+                'tool.arguments',
+                'tool.input',
+                'tool.args'
+            ) THEN attr_string END) as span_tool_arguments_raw,
+            max(CASE WHEN attr_key IN (
+                'gen_ai.tool.result',
+                'tool.result',
+                'tool.output'
+            ) THEN attr_string END) as span_tool_result_raw,
+            max(CASE WHEN attr_key = 'gen_ai.input.messages' THEN attr_string END) as input_messages_raw,
+            max(CASE WHEN attr_key = 'gen_ai.output.messages' THEN attr_string END) as output_messages_raw
+        FROM attrs
+        GROUP BY trace_id, span_id, span_name, duration_ms, start_ms, end_ms
+    ),
+    representative AS (
+        SELECT DISTINCT ON (trace_id)
+            trace_id,
+            span_id,
+            span_name,
+            session_id,
+            request_id,
+            user_id,
+            project_id,
+            agent_id,
+            agent_version,
+            source,
+            engine,
+            model,
+            start_ms,
+            end_ms,
+            duration_ms,
+            COALESCE(prompt_tokens, 0) as prompt_tokens,
+            COALESCE(completion_tokens, 0) as completion_tokens,
+            COALESCE(total_cost_usd, 0.0) as total_cost_usd,
+            error_type,
+            error_message,
+            span_tool_name,
+            span_tool_call_id,
+            span_tool_arguments_raw,
+            span_tool_result_raw,
+            input_messages_raw,
+            output_messages_raw
+        FROM extracted
+        ORDER BY
+            trace_id,
+            CASE WHEN output_messages_raw IS NOT NULL AND output_messages_raw != '' THEN 0 ELSE 1 END,
+            CASE WHEN input_messages_raw IS NOT NULL AND input_messages_raw != '' THEN 0 ELSE 1 END,
+            COALESCE(total_cost_usd, 0) DESC,
+            CASE
+                WHEN span_name = 'raw_gen_ai_request' THEN 0
+                WHEN span_name = 'litellm_request' THEN 1
+                ELSE 2
+            END,
+            start_ms DESC
+    ),
+    merged AS (
+        SELECT
+            e.trace_id,
+            r.span_id,
+            r.span_name,
+            COALESCE(MAX(CASE WHEN e.session_id IS NOT NULL AND e.session_id != '' THEN e.session_id END), r.session_id) AS session_id,
+            COALESCE(MAX(CASE WHEN e.request_id IS NOT NULL AND e.request_id != '' THEN e.request_id END), r.request_id) AS request_id,
+            COALESCE(MAX(CASE WHEN e.user_id IS NOT NULL AND e.user_id != '' THEN e.user_id END), r.user_id) AS user_id,
+            COALESCE(MAX(CASE WHEN e.project_id IS NOT NULL AND e.project_id != '' THEN e.project_id END), r.project_id) AS project_id,
+            COALESCE(MAX(CASE WHEN e.agent_id IS NOT NULL AND e.agent_id != '' THEN e.agent_id END), r.agent_id) AS agent_id,
+            COALESCE(MAX(CASE WHEN e.agent_version IS NOT NULL AND e.agent_version != '' THEN e.agent_version END), r.agent_version) AS agent_version,
+            COALESCE(MAX(CASE WHEN e.source IS NOT NULL AND e.source != '' THEN e.source END), r.source) AS source,
+            COALESCE(MAX(CASE WHEN e.engine IS NOT NULL AND e.engine != '' THEN e.engine END), r.engine) AS engine,
+            COALESCE(MAX(CASE WHEN e.model IS NOT NULL AND e.model != '' THEN e.model END), r.model) AS model,
+            MIN(e.start_ms) AS start_ms,
+            MAX(e.end_ms) AS end_ms,
+            MAX(e.duration_ms) AS duration_ms,
+            COALESCE(MAX(e.prompt_tokens), r.prompt_tokens, 0) AS prompt_tokens,
+            COALESCE(MAX(e.completion_tokens), r.completion_tokens, 0) AS completion_tokens,
+            COALESCE(MAX(e.total_cost_usd), r.total_cost_usd, 0.0) AS total_cost_usd,
+            CASE
+                WHEN SUM(CASE WHEN lower(coalesce(e.otel_status, '')) = 'error' OR e.error_type IS NOT NULL OR e.error_message IS NOT NULL THEN 1 ELSE 0 END) > 0 THEN 'error'
+                ELSE 'ok'
+            END AS status,
+            COALESCE(MAX(CASE WHEN e.error_type IS NOT NULL AND e.error_type != '' THEN e.error_type END), r.error_type) AS error_type,
+            COALESCE(MAX(CASE WHEN e.error_message IS NOT NULL AND e.error_message != '' THEN e.error_message END), r.error_message) AS error_message,
+            COALESCE(MAX(CASE WHEN e.span_tool_name IS NOT NULL AND e.span_tool_name != '' THEN e.span_tool_name END), r.span_tool_name) AS span_tool_name,
+            COALESCE(MAX(CASE WHEN e.span_tool_call_id IS NOT NULL AND e.span_tool_call_id != '' THEN e.span_tool_call_id END), r.span_tool_call_id) AS span_tool_call_id,
+            COALESCE(MAX(CASE WHEN e.span_tool_arguments_raw IS NOT NULL AND e.span_tool_arguments_raw != '' THEN e.span_tool_arguments_raw END), r.span_tool_arguments_raw) AS span_tool_arguments_raw,
+            COALESCE(MAX(CASE WHEN e.span_tool_result_raw IS NOT NULL AND e.span_tool_result_raw != '' THEN e.span_tool_result_raw END), r.span_tool_result_raw) AS span_tool_result_raw,
+            COALESCE(MAX(CASE WHEN e.input_messages_raw IS NOT NULL AND e.input_messages_raw != '' THEN e.input_messages_raw END), r.input_messages_raw) AS input_messages_raw,
+            COALESCE(MAX(CASE WHEN e.output_messages_raw IS NOT NULL AND e.output_messages_raw != '' THEN e.output_messages_raw END), r.output_messages_raw) AS output_messages_raw
+        FROM extracted e
+        JOIN representative r ON r.trace_id = e.trace_id
+        GROUP BY e.trace_id, r.span_id, r.span_name, r.session_id, r.request_id, r.user_id, r.project_id, r.agent_id, r.agent_version, r.source, r.engine, r.model, r.prompt_tokens, r.completion_tokens, r.total_cost_usd, r.error_type, r.error_message, r.span_tool_name, r.span_tool_call_id, r.span_tool_arguments_raw, r.span_tool_result_raw, r.input_messages_raw, r.output_messages_raw
+    )
+    SELECT
+        trace_id as model_call_id,
+        trace_id,
+        span_id,
+        span_name,
+        session_id,
+        CASE
+            WHEN session_id IS NOT NULL AND request_id IS NOT NULL THEN session_id || ':' || request_id
+            ELSE trace_id
+        END as turn_id,
+        request_id,
+        user_id,
+        project_id,
+        agent_id,
+        agent_version,
+        source,
+        engine,
+        model,
+        start_ms,
+        end_ms,
+        duration_ms,
+        prompt_tokens,
+        completion_tokens,
+        COALESCE(prompt_tokens, 0) + COALESCE(completion_tokens, 0) as total_tokens,
+        total_cost_usd,
+        status,
+        error_type,
+        error_message,
+        span_tool_name,
+        span_tool_call_id,
+        span_tool_arguments_raw,
+        span_tool_result_raw,
+        input_messages_raw,
+        output_messages_raw
+    FROM merged
+    """)
+
+    # Silver: model calls
+    conn.execute("""
+    INSERT INTO model_calls (
+        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,
+        user_id, project_id, agent_id, agent_version, source, engine, model,
+        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,
+        total_cost_usd, status, error_type, error_message,
+        input_messages_raw, output_messages_raw, created_at, updated_at
+    )
+    SELECT
+        model_call_id, trace_id, span_id, span_name, session_id, turn_id, request_id,
+        user_id, project_id, agent_id, agent_version, source, engine, model,
+        start_ms, end_ms, duration_ms, prompt_tokens, completion_tokens, total_tokens,
+        total_cost_usd, status, error_type, error_message,
+        input_messages_raw, output_messages_raw, now(), now()
+    FROM tmp_model_calls
+    ON CONFLICT(model_call_id) DO UPDATE SET
+        span_id = COALESCE(excluded.span_id, model_calls.span_id),
+        span_name = COALESCE(excluded.span_name, model_calls.span_name),
+        session_id = COALESCE(excluded.session_id, model_calls.session_id),
+        turn_id = COALESCE(excluded.turn_id, model_calls.turn_id),
+        request_id = COALESCE(excluded.request_id, model_calls.request_id),
+        user_id = COALESCE(excluded.user_id, model_calls.user_id),
+        project_id = COALESCE(excluded.project_id, model_calls.project_id),
+        agent_id = COALESCE(excluded.agent_id, model_calls.agent_id),
+        agent_version = COALESCE(excluded.agent_version, model_calls.agent_version),
+        source = COALESCE(excluded.source, model_calls.source),
+        engine = COALESCE(excluded.engine, model_calls.engine),
+        model = COALESCE(excluded.model, model_calls.model),
+        start_ms = COALESCE(excluded.start_ms, model_calls.start_ms),
+        end_ms = COALESCE(excluded.end_ms, model_calls.end_ms),
+        duration_ms = COALESCE(excluded.duration_ms, model_calls.duration_ms),
+        prompt_tokens = COALESCE(excluded.prompt_tokens, model_calls.prompt_tokens),
+        completion_tokens = COALESCE(excluded.completion_tokens, model_calls.completion_tokens),
+        total_tokens = COALESCE(excluded.total_tokens, model_calls.total_tokens),
+        total_cost_usd = COALESCE(excluded.total_cost_usd, model_calls.total_cost_usd),
+        status = COALESCE(excluded.status, model_calls.status),
+        error_type = COALESCE(excluded.error_type, model_calls.error_type),
+        error_message = COALESCE(excluded.error_message, model_calls.error_message),
+        input_messages_raw = COALESCE(excluded.input_messages_raw, model_calls.input_messages_raw),
+        output_messages_raw = COALESCE(excluded.output_messages_raw, model_calls.output_messages_raw),
+        updated_at = now()
+    """)
+
+    # Gold summary table used by existing dashboards
+    conn.execute("""
+    INSERT INTO spans (
+        trace_id, span_id, span_name, model, cost, input_tokens, output_tokens,
+        session_id, user_id, project_id, agent_id, engine, duration_ms, start_ms, created_at
+    )
+    SELECT
+        trace_id,
+        span_id,
+        span_name,
+        model,
+        total_cost_usd,
+        prompt_tokens,
+        completion_tokens,
+        session_id,
+        user_id,
+        project_id,
+        agent_id,
+        engine,
+        duration_ms,
+        start_ms,
+        now()
+    FROM tmp_model_calls
+    ON CONFLICT(trace_id, span_id) DO UPDATE SET
+        span_name = COALESCE(excluded.span_name, spans.span_name),
+        model = COALESCE(excluded.model, spans.model),
+        cost = COALESCE(excluded.cost, spans.cost),
+        input_tokens = COALESCE(excluded.input_tokens, spans.input_tokens),
+        output_tokens = COALESCE(excluded.output_tokens, spans.output_tokens),
+        session_id = COALESCE(excluded.session_id, spans.session_id),
+        user_id = COALESCE(excluded.user_id, spans.user_id),
+        project_id = COALESCE(excluded.project_id, spans.project_id),
+        agent_id = COALESCE(excluded.agent_id, spans.agent_id),
+        engine = COALESCE(excluded.engine, spans.engine),
+        duration_ms = COALESCE(excluded.duration_ms, spans.duration_ms),
+        start_ms = COALESCE(excluded.start_ms, spans.start_ms)
+    """)
+
+    # Turns from model calls
+    conn.execute("""
+    INSERT INTO turns (
+        turn_id, session_id, request_id, turn_index, started_at, ended_at, status,
+        model, provider, prompt_tokens, completion_tokens, total_tokens, cost_usd,
+        trace_id, created_at, updated_at
+    )
+    SELECT
+        turn_id,
+        session_id,
+        request_id,
+        NULL,
+        to_timestamp(start_ms / 1000.0),
+        to_timestamp(end_ms / 1000.0),
+        status,
+        model,
+        engine,
+        prompt_tokens,
+        completion_tokens,
+        total_tokens,
+        total_cost_usd,
+        trace_id,
+        now(),
+        now()
+    FROM tmp_model_calls
+    ON CONFLICT(turn_id) DO UPDATE SET
+        session_id = excluded.session_id,
+        request_id = excluded.request_id,
+        started_at = LEAST(turns.started_at, excluded.started_at),
+        ended_at = GREATEST(turns.ended_at, excluded.ended_at),
+        status = CASE WHEN excluded.status = 'error' THEN 'error' ELSE turns.status END,
+        model = excluded.model,
+        provider = excluded.provider,
+        prompt_tokens = COALESCE(excluded.prompt_tokens, turns.prompt_tokens),
+        completion_tokens = COALESCE(excluded.completion_tokens, turns.completion_tokens),
+        total_tokens = COALESCE(excluded.total_tokens, turns.total_tokens),
+        cost_usd = COALESCE(excluded.cost_usd, turns.cost_usd),
+        trace_id = excluded.trace_id,
+        updated_at = now()
+    """)
+
+    # Parse structured tool calls from raw message payloads.
+    extract_messages_and_tools(conn)
+
+    # Rebuild session rollups for sessions touched in this file.
+    conn.execute("DROP TABLE IF EXISTS tmp_touched_sessions")
+    conn.execute("""
+    CREATE TEMP TABLE tmp_touched_sessions AS
+    SELECT DISTINCT session_id
+    FROM tmp_model_calls
+    WHERE session_id IS NOT NULL AND session_id != ''
+    """)
+
+    conn.execute("""
+    INSERT INTO sessions (
+        session_id, user_id, project_id, agent_id, agent_version, source, engine,
+        started_at, ended_at, status, total_turns, total_requests,
+        total_cost_usd, total_tokens, created_at, updated_at
+    )
+    SELECT
+        mc.session_id,
+        any_value(mc.user_id) as user_id,
+        any_value(mc.project_id) as project_id,
+        any_value(mc.agent_id) as agent_id,
+        any_value(mc.agent_version) as agent_version,
+        any_value(mc.source) as source,
+        any_value(mc.engine) as engine,
+        to_timestamp(min(mc.start_ms) / 1000.0) as started_at,
+        to_timestamp(max(mc.end_ms) / 1000.0) as ended_at,
+        CASE WHEN sum(CASE WHEN mc.status = 'error' THEN 1 ELSE 0 END) > 0 THEN 'error' ELSE 'ok' END as status,
+        count(distinct mc.turn_id) as total_turns,
+        count(*) as total_requests,
+        sum(coalesce(mc.total_cost_usd, 0.0)) as total_cost_usd,
+        sum(coalesce(mc.total_tokens, 0)) as total_tokens,
+        now(),
+        now()
+    FROM model_calls mc
+    JOIN tmp_touched_sessions t ON t.session_id = mc.session_id
+    GROUP BY mc.session_id
+    ON CONFLICT(session_id) DO UPDATE SET
+        user_id = excluded.user_id,
+        project_id = excluded.project_id,
+        agent_id = excluded.agent_id,
+        agent_version = excluded.agent_version,
+        source = excluded.source,
+        engine = excluded.engine,
+        started_at = excluded.started_at,
+        ended_at = excluded.ended_at,
+        status = excluded.status,
+        total_turns = excluded.total_turns,
+        total_requests = excluded.total_requests,
+        total_cost_usd = excluded.total_cost_usd,
+        total_tokens = excluded.total_tokens,
+        updated_at = now()
+    """)
+
+    # Recompute per-session ordering for turns.
+    conn.execute("""
+    UPDATE turns AS t
+    SET turn_index = ranked.turn_index
+    FROM (
+        SELECT
+            turn_id,
+            row_number() OVER (PARTITION BY session_id ORDER BY started_at, turn_id) - 1 AS turn_index
+        FROM turns
+        WHERE session_id IN (SELECT session_id FROM tmp_touched_sessions)
+    ) ranked
+    WHERE t.turn_id = ranked.turn_id
+    """)
+
+
+def upsert_processed_file(conn, jsonl_file: Path) -> None:
+    mtime_ms, size_bytes = get_file_stats(jsonl_file)
+    conn.execute(
+        """
+        INSERT INTO processed_files (file_path, mtime_ms, size_bytes, processed_at)
+        VALUES (?, ?, ?, now())
+        ON CONFLICT(file_path) DO UPDATE SET
+            mtime_ms = excluded.mtime_ms,
+            size_bytes = excluded.size_bytes,
+            processed_at = excluded.processed_at
+        """,
+        [str(jsonl_file), mtime_ms, size_bytes],
+    )
+
+
+def main() -> int:
+    """Main entry point."""
+    traces_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else None
+    db_path = Path(sys.argv[2]) if len(sys.argv) > 2 else None
+
+    traces_dir = get_traces_dir(str(traces_dir) if traces_dir else None)
+    db_path = get_db_path(traces_dir, str(db_path) if db_path else None)
+
+    if not traces_dir.exists():
+        log.error(f"Traces directory not found: {traces_dir}")
+        print(json.dumps({
+            'status': 'error',
+            'indexed': False,
+            'message': f'Traces directory not found: {traces_dir}',
+        }))
+        return 1
+
+    lock_handle = try_acquire_index_lock(get_lock_file(traces_dir))
+    if lock_handle is None:
+        log.info('Indexing already in progress')
+        print(json.dumps({
+            'status': 'busy',
+            'indexed': False,
+            'message': 'Indexing already in progress',
+        }))
+        return 3
+
+    try:
+        jsonl_files = list(traces_dir.glob('*.jsonl'))
+        if not jsonl_files:
+            log.info(f"No JSONL files found in {traces_dir}")
+            print(json.dumps({
+                'status': 'ok',
+                'indexed': False,
+                'message': 'No trace files to index',
+            }))
+            return 2
+
+        log.debug(f"Found {len(jsonl_files)} JSONL files")
+
+        state_file = get_state_file(traces_dir)
+        state = load_index_state(state_file)
+
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            import duckdb
+        except ImportError:
+            log.error("duckdb module not found. Install with: pip install duckdb")
+            print(json.dumps({
+                'status': 'error',
+                'indexed': False,
+                'message': 'duckdb module not found',
+            }), file=sys.stderr)
+            return 1
+
+        try:
+            log.debug(f"Connecting to database: {db_path}")
+            conn = duckdb.connect(str(db_path))
+            ensure_schema(conn)
+
+            changed_files = get_changed_files(conn, traces_dir)
+            if not changed_files:
+                current_mtime = get_latest_jsonl_mtime(traces_dir)
+                state['version'] = 2
+                state['indexed'] = True
+                state['lastMtime'] = current_mtime
+                state['duckdbPath'] = str(db_path)
+                state['indexedAt'] = datetime.utcnow().isoformat() + 'Z'
+                save_index_state(state_file, state)
+
+                log.info("Index is up-to-date; no changed files")
+                print(json.dumps({
+                    'status': 'ok',
+                    'indexed': True,
+                    'message': 'Already indexed',
+                }))
+                conn.close()
+                return 2
+
+            log.info(f"Incremental indexing {len(changed_files)} changed file(s)")
+
+            for jsonl_file in changed_files:
+                log.debug(f"Indexing file: {jsonl_file}")
+                ingest_single_jsonl_file(conn, jsonl_file)
+                upsert_processed_file(conn, jsonl_file)
+
+            current_mtime = get_latest_jsonl_mtime(traces_dir)
+            state['version'] = 2
+            state['indexed'] = True
+            state['lastMtime'] = current_mtime
+            state['duckdbPath'] = str(db_path)
+            state['indexedAt'] = datetime.utcnow().isoformat() + 'Z'
+            save_index_state(state_file, state)
+
+            count_result = conn.execute("SELECT COUNT(*) FROM model_calls").fetchall()
+            total_records = count_result[0][0] if count_result else 0
+            log.debug(f"Ingestion complete. Total model_calls: {total_records}")
+
+            conn.close()
+
+            log.info(f"Indexing complete: {db_path}")
+            print(json.dumps({
+                'status': 'ok',
+                'indexed': True,
+                'message': f'Indexed {len(changed_files)} changed file(s) at {state["indexedAt"]}',
+            }))
+            return 0
+
+        except Exception as e:
+            log.error(f"Indexing failed: {e}")
+            print(json.dumps({
+                'status': 'error',
+                'indexed': False,
+                'message': str(e),
+            }), file=sys.stderr)
+            return 1
+    finally:
+        release_index_lock(lock_handle)
+
+
+if __name__ == '__main__':
+    sys.exit(main())