@agenthippo-ai/agenthippo 0.1.0

package/out/scripts/analytics/query-analytics.ps1

@@ -0,0 +1,396 @@
+#
+# Query AgentHippo Analytics with DuckDB (Windows)
+#
+# Usage:
+#   .\query-analytics.ps1                       # Interactive DuckDB shell
+#   .\query-analytics.ps1 -Cost                 # Show total costs
+#   .\query-analytics.ps1 -CostBySession        # Costs grouped by session
+#   .\query-analytics.ps1 -CostByModel          # Costs grouped by model
+#   .\query-analytics.ps1 -Session <id>         # Show specific session
+#   .\query-analytics.ps1 -LatestSession        # Latest session chat + cost
+#   .\query-analytics.ps1 -Sql "SELECT..."      # Run custom SQL
+#
+
+param(
+    [switch]$Cost,
+    [switch]$CostBySession,
+    [switch]$CostByModel,
+    [string]$Session = "",
+    [switch]$LatestSession,
+    [string]$Sql = "",
+    [switch]$Help
+)
+
+$ErrorActionPreference = "Stop"
+
+$AgentHomeDir = if ($env:AGENT_HOME_DIR) { $env:AGENT_HOME_DIR } else { Join-Path $env:USERPROFILE ".agent-hippo" }
+$AnalyticsDir = Join-Path $AgentHomeDir "analytics"
+$BinDir = Join-Path $AgentHomeDir "bin"
+$TracesDir = Join-Path $AnalyticsDir "traces"
+
+# Find DuckDB binary
+function Find-DuckDB {
+    $systemDuck = Get-Command duckdb -ErrorAction SilentlyContinue
+    if ($systemDuck) { return $systemDuck.Source }
+
+    $localDuck = Join-Path $BinDir "duckdb.exe"
+    if (Test-Path $localDuck) { return $localDuck }
+
+    Write-Host "[ERROR] DuckDB not found. Run 'Setup Analytics' first." -ForegroundColor Red
+    exit 1
+}
+
+$DuckDB = Find-DuckDB
+
+# Check if traces exist
+function Check-Traces {
+    $files = Get-ChildItem $TracesDir -Filter "*.jsonl" -ErrorAction SilentlyContinue
+    if (-not $files -or $files.Count -eq 0) {
+        Write-Host "No trace files found in $TracesDir/"
+        Write-Host "Make sure OTEL Collector is running and LiteLLM has the 'otel' callback enabled."
+        exit 0
+    }
+}
+
+# Escape path for SQL (backslashes in Windows paths)
+$TracesGlob = ($TracesDir -replace '\\', '/') + "/*.jsonl"
+
+# Base CTE query to parse OTEL JSONL format
+$BaseQuery = @"
+WITH raw_spans AS (
+    SELECT
+        unnest(resourceSpans) as rs
+    FROM read_json('$TracesGlob',
+                   format='newline_delimited',
+                   ignore_errors=true)
+),
+spans AS (
+    SELECT
+        unnest(rs.scopeSpans) as ss
+    FROM raw_spans
+),
+flat_spans AS (
+    SELECT
+        unnest(ss.spans) as span
+    FROM spans
+),
+parsed AS (
+    SELECT
+        span.traceId as trace_id,
+        span.spanId as span_id,
+        span.name as span_name,
+        span.startTimeUnixNano::BIGINT / 1000000 as start_ms,
+        span.endTimeUnixNano::BIGINT / 1000000 as end_ms,
+        (span.endTimeUnixNano::BIGINT - span.startTimeUnixNano::BIGINT) / 1000000 as duration_ms,
+        json_extract(to_json(span), '$.attributes') as attrs_json
+    FROM flat_spans
+),
+span_filter AS (
+    SELECT
+        trace_id,
+        span_name,
+        duration_ms,
+        start_ms,
+        attrs_json
+    FROM parsed
+    WHERE span_name = 'Received Proxy Server Request'
+       OR EXISTS (
+           SELECT 1
+           FROM json_each(coalesce(attrs_json, '[]'::JSON)) as attr
+           WHERE json_extract_string(attr.value, '$.key') = 'gen_ai.cost.total_cost'
+       )
+),
+attrs AS (
+    SELECT
+        trace_id,
+        span_name,
+        duration_ms,
+        start_ms,
+        json_extract_string(value, '$.key') as attr_key,
+        json_extract_string(value, '$.value.stringValue') as attr_string,
+        json_extract_string(value, '$.value.intValue') as attr_int,
+        json_extract(value, '$.value.doubleValue') as attr_double
+    FROM span_filter,
+         json_each(coalesce(attrs_json, '[]'::JSON))
+)
+"@
+
+# Extract LLM attributes from OTEL span attributes
+$ExtractAttrs = @"
+SELECT
+    trace_id,
+    span_name,
+    duration_ms,
+    start_ms,
+    COALESCE(
+        regexp_extract(
+            max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END),
+            'x-litellm-metadata-session-id'': ''([^'']+)''', 1
+        ),
+        regexp_extract(
+            max(CASE WHEN attr_key = 'metadata.user_api_key_end_user_id' THEN attr_string END),
+            'session_([a-f0-9-]+)', 1
+        ),
+        regexp_extract(
+            max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END),
+            'session_id'': ''([^'']+)''', 1
+        )
+    ) as session_id,
+    COALESCE(
+        regexp_extract(
+            max(CASE WHEN attr_key = 'metadata.requester_metadata' THEN attr_string END),
+            'user_id'': ''([^'']+)''', 1
+        ),
+        regexp_extract(
+            max(CASE WHEN attr_key = 'metadata.requester_custom_headers' THEN attr_string END),
+            'x-litellm-metadata-user-id'': ''([^'']+)''', 1
+        ),
+        max(CASE WHEN attr_key = 'metadata.user_api_key_end_user_id' THEN attr_string END)
+    ) as user_id,
+    max(CASE WHEN attr_key IN ('gen_ai.request.model', 'gen_ai.response.model', 'model') THEN attr_string END) as model,
+    max(CASE WHEN attr_key = 'gen_ai.cost.total_cost' THEN CAST(attr_double AS DOUBLE) END) as cost,
+    max(CASE WHEN attr_key = 'gen_ai.usage.input_tokens' THEN CAST(attr_int AS BIGINT) END) as input_tokens,
+    max(CASE WHEN attr_key = 'gen_ai.usage.output_tokens' THEN CAST(attr_int AS BIGINT) END) as output_tokens
+FROM attrs
+GROUP BY trace_id, span_name, duration_ms, start_ms
+"@
+
+function Run-Query {
+    param([string]$QuerySql)
+    & $DuckDB -c $QuerySql
+}
+
+# --- Total Cost ---
+function Query-TotalCost {
+    Check-Traces
+    Write-Host "Total Cost Summary" -ForegroundColor Cyan
+    Write-Host "===================="
+    Run-Query @"
+    $BaseQuery,
+    extracted AS ($ExtractAttrs)
+    SELECT
+        COALESCE(SUM(cost), 0) as total_cost,
+        COALESCE(SUM(input_tokens), 0) as total_input_tokens,
+        COALESCE(SUM(output_tokens), 0) as total_output_tokens,
+        COUNT(*) as total_requests,
+        ROUND(AVG(duration_ms), 0) as avg_latency_ms
+    FROM extracted;
+"@
+}
+
+# --- Cost by Session ---
+function Query-CostBySession {
+    Check-Traces
+    Write-Host "Cost by Session" -ForegroundColor Cyan
+    Write-Host "=================="
+    Run-Query @"
+    $BaseQuery,
+    extracted AS ($ExtractAttrs)
+    SELECT
+        COALESCE(session_id, '(no session)') as session_id,
+        ROUND(COALESCE(SUM(cost), 0), 6) as total_cost,
+        COALESCE(SUM(input_tokens), 0) as input_tokens,
+        COALESCE(SUM(output_tokens), 0) as output_tokens,
+        COUNT(*) as requests,
+        ROUND(AVG(duration_ms), 0) as avg_latency_ms
+    FROM extracted
+    GROUP BY session_id
+    ORDER BY total_cost DESC
+    LIMIT 20;
+"@
+}
+
+# --- Cost by Model ---
+function Query-CostByModel {
+    Check-Traces
+    Write-Host "Cost by Model" -ForegroundColor Cyan
+    Write-Host "================"
+    Run-Query @"
+    $BaseQuery,
+    extracted AS ($ExtractAttrs)
+    SELECT
+        COALESCE(model, '(unknown)') as model,
+        ROUND(COALESCE(SUM(cost), 0), 6) as total_cost,
+        COALESCE(SUM(input_tokens), 0) as input_tokens,
+        COALESCE(SUM(output_tokens), 0) as output_tokens,
+        COUNT(*) as requests,
+        ROUND(AVG(duration_ms), 0) as avg_latency_ms
+    FROM extracted
+    GROUP BY model
+    ORDER BY total_cost DESC;
+"@
+}
+
+# --- Specific Session ---
+function Query-SessionById {
+    param([string]$SessionId)
+    Check-Traces
+    Write-Host "Session: $SessionId" -ForegroundColor Cyan
+    Write-Host "========================="
+    Run-Query @"
+    $BaseQuery,
+    extracted AS ($ExtractAttrs)
+    SELECT
+        model,
+        ROUND(cost, 6) as cost,
+        input_tokens,
+        output_tokens,
+        duration_ms as latency_ms,
+        to_timestamp(start_ms / 1000) as timestamp
+    FROM extracted
+    WHERE session_id = '$SessionId'
+    ORDER BY start_ms;
+"@
+}
+
+# --- Latest Session ---
+function Query-LatestSession {
+    Check-Traces
+    Write-Host "Latest Session" -ForegroundColor Cyan
+    Write-Host "================"
+    Write-Host ""
+
+    # Summary table
+    Run-Query @"
+    $BaseQuery,
+    extracted AS ($ExtractAttrs)
+    SELECT
+        COALESCE(session_id, '(no session)') as session_id,
+        COALESCE(model, '(unknown)') as model,
+        ROUND(COALESCE(cost, 0), 6) as cost,
+        COALESCE(input_tokens, 0) as input_tokens,
+        COALESCE(output_tokens, 0) as output_tokens,
+        duration_ms as latency_ms,
+        to_timestamp(start_ms / 1000) as timestamp
+    FROM extracted
+    ORDER BY start_ms DESC
+    LIMIT 1;
+"@
+
+    # Try to format messages using Python (available on most dev machines)
+    $pythonCmd = if (Get-Command python3 -ErrorAction SilentlyContinue) { "python3" }
+                 elseif (Get-Command python -ErrorAction SilentlyContinue) { "python" }
+                 else { $null }
+
+    if ($pythonCmd) {
+        Write-Host ""
+        Write-Host "Messages" -ForegroundColor Cyan
+        Write-Host "==========="
+
+        $msgQuery = @"
+        $BaseQuery
+        SELECT
+            max(CASE WHEN attr_key IN ('gen_ai.input.messages', 'llm.None.messages', 'llm.openai.messages', 'llm.anthropic.messages') THEN attr_string END) as input_messages,
+            max(CASE WHEN attr_key IN ('gen_ai.output.messages', 'llm.openai.choices', 'llm.anthropic.completion') THEN attr_string END) as output_messages
+        FROM attrs
+        WHERE trace_id = (
+            SELECT trace_id FROM (
+                $BaseQuery,
+                extracted AS ($ExtractAttrs)
+                SELECT trace_id, start_ms FROM extracted ORDER BY start_ms DESC LIMIT 1
+            )
+        )
+        GROUP BY trace_id;
+"@
+
+        $pythonScript = @'
+import sys, json, textwrap
+
+def fmt(content, indent=2):
+    if not content: return "  (empty)"
+    if len(content) > 2000: content = content[:2000] + "... (truncated)"
+    return textwrap.fill(content, width=100, initial_indent=" "*indent, subsequent_indent=" "*indent)
+
+def extract(msg):
+    if isinstance(msg, str): return msg
+    if not isinstance(msg, dict): return str(msg)
+    if "content" in msg:
+        c = msg["content"]
+        if isinstance(c, str): return c
+        if isinstance(c, list):
+            parts = []
+            for item in c:
+                if isinstance(item, dict):
+                    if item.get("type") in ("thinking", "signature"): continue
+                    t = item.get("text", item.get("content", ""))
+                    if t: parts.append(t)
+                elif isinstance(item, str): parts.append(item)
+            return "\n".join(parts) if parts else "(no content)"
+        return str(c)
+    if "message" in msg: return extract(msg["message"])
+    return str(msg)
+
+try:
+    data = sys.stdin.read().strip()
+    if not data: print("(no messages found)"); sys.exit(0)
+    rows = json.loads(data)
+    if not rows: print("(no messages found)"); sys.exit(0)
+    row = rows[0]
+    for label, key in [("INPUT", "input_messages"), ("OUTPUT", "output_messages")]:
+        print(f"\n--- {label} ---")
+        raw = row.get(key, "")
+        if not raw: print("  (none)"); continue
+        try:
+            msgs = json.loads(raw)
+            if isinstance(msgs, list):
+                for i, m in enumerate(msgs):
+                    if isinstance(m, dict):
+                        role = m.get("role", "unknown")
+                        print(f"\n  [{i}] {role.upper()}")
+                        print(fmt(extract(m)))
+            elif isinstance(msgs, dict):
+                print(f"\n  {msgs.get('role','assistant').upper()}")
+                print(fmt(extract(msgs)))
+        except json.JSONDecodeError:
+            print(fmt(raw))
+except Exception as e:
+    print(f"Error: {e}", file=sys.stderr)
+'@
+
+        & $DuckDB -json -c $msgQuery | & $pythonCmd -c $pythonScript
+    }
+}
+
+# --- Interactive Shell ---
+function Start-InteractiveShell {
+    Check-Traces
+    Write-Host "DuckDB Interactive Shell" -ForegroundColor Cyan
+    Write-Host "==========================="
+    Write-Host ""
+    Write-Host "Trace files: $TracesDir\*.jsonl"
+    Write-Host ""
+    Write-Host "Example queries:"
+    Write-Host "  SELECT * FROM read_json('$TracesGlob') LIMIT 5;"
+    Write-Host ""
+    & $DuckDB
+}
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if ($Help) {
+    Write-Host "Usage: .\query-analytics.ps1 [options]"
+    Write-Host ""
+    Write-Host "Options:"
+    Write-Host "  (none)             Interactive DuckDB shell"
+    Write-Host "  -Cost              Show total costs"
+    Write-Host "  -CostBySession     Costs grouped by session"
+    Write-Host "  -CostByModel       Costs grouped by model"
+    Write-Host "  -Session <id>      Show specific session details"
+    Write-Host "  -LatestSession     Show latest session chat + cost"
+    Write-Host "  -Sql `"...`"        Run custom SQL query"
+    Write-Host "  -Help              Show this help"
+    exit 0
+}
+
+if ($Cost) { Query-TotalCost; exit 0 }
+if ($CostBySession) { Query-CostBySession; exit 0 }
+if ($CostByModel) { Query-CostByModel; exit 0 }
+if ($Session) { Query-SessionById -SessionId $Session; exit 0 }
+if ($LatestSession) { Query-LatestSession; exit 0 }
+if ($Sql) { Check-Traces; Run-Query $Sql; exit 0 }
+
+# Default: interactive shell
+Start-InteractiveShell
+