@senad-d/observme 0.1.0

package/dashboards/observme-logs-llm.json

@@ -0,0 +1,644 @@
+{
+  "__requires": [
+    {
+      "id": "grafana",
+      "name": "Grafana",
+      "type": "grafana",
+      "version": "10.0.0"
+    },
+    {
+      "id": "prometheus",
+      "name": "Prometheus",
+      "type": "datasource",
+      "version": "1.0.0"
+    },
+    {
+      "id": "loki",
+      "name": "Loki",
+      "type": "datasource",
+      "version": "1.0.0"
+    },
+    {
+      "id": "timeseries",
+      "name": "Time series",
+      "type": "panel",
+      "version": ""
+    },
+    {
+      "id": "row",
+      "name": "Row",
+      "type": "panel",
+      "version": ""
+    },
+    {
+      "id": "logs",
+      "name": "Logs",
+      "type": "panel",
+      "version": ""
+    },
+    {
+      "id": "text",
+      "name": "Text",
+      "type": "panel",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [
+    {
+      "targetBlank": false,
+      "title": "Open canonical LLM Conversations",
+      "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}&${provider:queryparam}&${model:queryparam}&${content_kind:queryparam}"
+    },
+    {
+      "targetBlank": false,
+      "title": "Open matching Trace Journey",
+      "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}"
+    }
+  ],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 100,
+      "panels": [],
+      "title": "Aggregate LLM telemetry",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "LLM prompt/response p95 size trends measured in characters and filterable by provider/model. These *_size_chars histograms work even when content capture is disabled and must not be interpreted as bytes. No data means matching histogram samples are absent, not that size is zero.",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "11.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(observme_prompt_size_chars_bucket{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__rate_interval])))",
+          "legendFormat": "prompt p95",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(observme_response_size_chars_bucket{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__rate_interval])))",
+          "legendFormat": "response p95",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "LLM input/output size p95 (characters)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "Interval token deltas by selected provider/model for input, output, total, reasoning, and cache token families. Each point uses increase(...[$__interval]); optional reasoning/cache families may show no data when a provider does not report them.",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "sum"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "11.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_input_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "input {{provider}}/{{model}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_output_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "output {{provider}}/{{model}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_total_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "total {{provider}}/{{model}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_reasoning_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "reasoning {{provider}}/{{model}}",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_cache_read_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "cache read {{provider}}/{{model}}",
+          "range": true,
+          "refId": "E"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_cache_write_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "cache write {{provider}}/{{model}}",
+          "range": true,
+          "refId": "F"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(observme_llm_cache_write_1h_tokens_total{provider=~\"${provider:regex}\", model=~\"${model:regex}\"}[$__interval])) by (provider, model)",
+          "legendFormat": "cache write 1h {{provider}}/{{model}}",
+          "range": true,
+          "refId": "G"
+        }
+      ],
+      "title": "LLM token trends by type",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 101,
+      "panels": [],
+      "title": "Privacy-safe content routing",
+      "type": "row"
+    },
+    {
+      "datasource": null,
+      "description": "Visible privacy and empty-state guidance for optional redacted LLM content. This dashboard keeps aggregate telemetry and content-free lifecycle logs here, while prompt, response, and thinking bodies remain in the canonical conversation view.",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 10
+      },
+      "id": 3,
+      "links": [
+        {
+          "targetBlank": false,
+          "title": "Open canonical LLM Conversations",
+          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}&${provider:queryparam}&${model:queryparam}&${content_kind:queryparam}"
+        }
+      ],
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "## Canonical LLM conversation drill-down\n\nPrompts, responses, and thinking are redacted, opt-in content. Open **ObservMe LLM Conversations** with the current session/workflow/agent/run/provider/model/content-kind filters for the canonical timeline. Empty content can mean capture is disabled or no matching events exist. Explicit unsafe capture can expose sensitive text when redaction is disabled. Do not query by raw prompt, response, or thinking text.",
+        "mode": "markdown"
+      },
+      "pluginVersion": "11.1.0",
+      "title": "Canonical LLM conversation drill-down",
+      "type": "text"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "id": 102,
+      "panels": [],
+      "title": "Content-free lifecycle logs",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "description": "Newest-first LLM request lifecycle logs filtered by session, workflow, agent, run, provider, and model labels, capped at 500 lines. Structured labels remain available in log details. Use canonical LLM Conversations for redacted prompt/response/thinking bodies and Trace Journey for waterfalls.",
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "editorMode": "code",
+          "expr": "{service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", gen_ai_provider_name=~\"${provider:regex}\", gen_ai_request_model=~\"${model:regex}\", event_name=~\"llm[.]request[.](started|completed|failed)\", event_category!=\"llm_content\"}",
+          "maxLines": 500,
+          "queryType": "range",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "pluginVersion": "11.1.0",
+      "title": "LLM request logs",
+      "type": "logs",
+      "links": [
+        {
+          "targetBlank": false,
+          "title": "Open canonical LLM Conversations",
+          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}&${provider:queryparam}&${model:queryparam}&${content_kind:queryparam}"
+        },
+        {
+          "targetBlank": false,
+          "title": "Open matching Trace Journey",
+          "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}"
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "description": "Newest-first non-content ObservMe logs for the selected session/workflow/agent/run filters, capped at 500 lines. Structured labels remain available in log details. Provider/model filters intentionally apply only to the LLM request panel because general lifecycle records may not carry those labels. This panel excludes llm_content and tool_content so captured bodies remain in canonical content views; do not query by raw content.",
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 5,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "editorMode": "code",
+          "expr": "{service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", event_category!~\"llm_content|tool_content\"}",
+          "maxLines": 500,
+          "queryType": "range",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "pluginVersion": "11.1.0",
+      "title": "Session logs",
+      "type": "logs",
+      "links": [
+        {
+          "targetBlank": false,
+          "title": "Open canonical LLM Conversations",
+          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}&${provider:queryparam}&${model:queryparam}&${content_kind:queryparam}"
+        },
+        {
+          "targetBlank": false,
+          "title": "Open matching Trace Journey",
+          "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}"
+        }
+      ]
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "observme",
+    "pi",
+    "logs",
+    "llm"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\"}, pi_session_id)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Session ID",
+        "multi": true,
+        "name": "session_id",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\"}, pi_session_id)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\"}, pi_workflow_id)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Workflow ID",
+        "multi": true,
+        "name": "workflow_id",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\"}, pi_workflow_id)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\"}, pi_agent_id)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Agent ID",
+        "multi": true,
+        "name": "agent_id",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\"}, pi_agent_id)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\"}, pi_agent_run_id)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Agent run ID",
+        "multi": true,
+        "name": "agent_run_id",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\"}, pi_agent_run_id)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\"}, gen_ai_provider_name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Provider",
+        "multi": true,
+        "name": "provider",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\"}, gen_ai_provider_name)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", gen_ai_provider_name=~\"${provider:regex}\"}, gen_ai_request_model)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Model",
+        "multi": true,
+        "name": "model",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_name=~\"llm[.]request[.](started|completed|failed)|llm[.](prompt|response|thinking)[.]captured\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", gen_ai_provider_name=~\"${provider:regex}\"}, gen_ai_request_model)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({service_name=\"observme-pi-extension\", event_category=\"llm_content\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", gen_ai_provider_name=~\"${provider:regex}\", gen_ai_request_model=~\"${model:regex}\"}, pi_llm_content_kind)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Content kind",
+        "multi": true,
+        "name": "content_kind",
+        "options": [],
+        "query": "label_values({service_name=\"observme-pi-extension\", event_category=\"llm_content\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", gen_ai_provider_name=~\"${provider:regex}\", gen_ai_request_model=~\"${model:regex}\"}, pi_llm_content_kind)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "ObservMe Logs and LLM I/O",
+  "uid": "observme-logs-llm",
+  "version": 3,
+  "weekStart": ""
+}