@agent-native/core 0.97.0 → 0.98.0

package/corpus/templates/analytics/docs/schemas/first-party-analytics.md

@@ -106,10 +106,13 @@ Events are stored in `analytics_events`. Common query columns include:
 
 ## LLM observability events
 
-Core emits LLM usage as first-party events with:
+Core emits LLM usage, explicit user feedback, and optional inferred message
+sentiment as first-party events with:
 
 ```txt
 event_name = '$ai_generation'
+event_name = '$ai_feedback'
+event_name = '$ai_sentiment'
 ```
 
 Useful query fields live in `properties`:
@@ -129,8 +132,39 @@ Useful query fields live in `properties`:
 | `tool_calls`, `successful_tools`, `failed_tools` | Tool-call counts                          |
 | `$ai_is_error`, `status`, `$ai_error`            | Error status and message, when applicable |
 
-Install the `agent-observability-llm` dashboard template for canned panels over
-these events.
+Explicit thumbs feedback is content-free and uses these `$ai_feedback`
+properties:
+
+| Property              | Description                                    |
+| --------------------- | ---------------------------------------------- |
+| `sentiment`           | Explicit `positive` or `negative` user rating  |
+| `feedback_type`       | Feedback control/type that produced the rating |
+| `$ai_model`, `model`  | Model that generated the rated agent response  |
+| `run_id`, `thread_id` | Related agent run and thread identifiers       |
+
+These values describe user-provided thumbs feedback.
+
+When optional inferred sentiment is enabled, content-free `$ai_sentiment`
+events use these properties:
+
+| Property              | Description                                                  |
+| --------------------- | ------------------------------------------------------------ |
+| `sentiment`           | Inferred `positive`, `neutral`, or `negative` classification |
+| `method`              | Classification method; `llm` for model-inferred sentiment    |
+| `$ai_model`, `model`  | Main model attributed to the preceding agent response        |
+| `classifier_model`    | Small model used only to classify sentiment                  |
+| `run_id`, `thread_id` | Related agent run and thread identifiers                     |
+
+The inferred-sentiment event does not contain message text. Keep it separate
+from `$ai_feedback`: inferred sentiment is a model classification, while
+feedback sentiment is an explicit user rating. For by-model reporting, group
+on `$ai_model` or `model`; use `classifier_model` only to audit classifier usage.
+
+Agent Native observability panels belong in the canonical Agent Native dashboard
+(`agent-native-templates-first-party`), including generation metrics, explicit
+feedback sentiment, optional inferred message sentiment, and separate
+by-main-model breakdowns. Do not publish or install a separate observability
+dashboard template.
 
 Example dashboard panel:
 

package/corpus/templates/analytics/seeds/dashboards/agent-native-templates-first-party.json

@@ -1,6 +1,6 @@
 {
   "name": "Agent Native Templates (First-party)",
-  "description": "Template engagement and per-app activity recorded through analytics.agent-native.com/track. Source: first-party analytics_events.",
+  "description": "Template engagement and per-app activity recorded through analytics.agent-native.com/track. Source: first-party analytics_events. Includes LLM generation volume, latency, errors, token usage, cost, explicit user feedback, and optional inferred message sentiment from first-party tracking events.",
   "filters": [
     {
       "id": "timeRange",
@@ -755,6 +755,344 @@
         "yFormatter": "number",
         "description": "Distinct users who drove at least one referred signup in the window (distinct referrer_user on signups in the window)."
       }
+    },
+    {
+      "id": "llm-generations-30d",
+      "title": "LLM Generations (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COUNT(*) AS count FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "count",
+        "yFormatter": "number",
+        "description": "Agent LLM generation events emitted by core observability tracking over the last 30 days."
+      }
+    },
+    {
+      "id": "llm-cost-30d",
+      "title": "LLM Cost (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COALESCE(SUM(COALESCE(NULLIF(properties::jsonb ->> '$ai_total_cost_usd', '')::float, NULLIF(properties::jsonb ->> 'cost_usd', '')::float, COALESCE(NULLIF(properties::jsonb ->> 'cost_cents_x100', '')::float, 0) / 10000.0)), 0) AS cost_usd FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "cost_usd",
+        "yFormatter": "currency",
+        "description": "Estimated provider spend from tracked LLM generation events."
+      }
+    },
+    {
+      "id": "llm-avg-latency-30d",
+      "title": "Avg LLM Latency (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COALESCE(AVG(COALESCE(NULLIF(properties::jsonb ->> 'duration_ms', '')::float, NULLIF(properties::jsonb ->> '$ai_latency', '')::float * 1000.0)), 0) AS latency_ms FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "latency_ms",
+        "yFormatter": "duration",
+        "description": "Average agent loop latency for tracked LLM generations."
+      }
+    },
+    {
+      "id": "llm-error-rate-30d",
+      "title": "LLM Error Rate (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT CASE WHEN COUNT(*) = 0 THEN 0 ELSE COUNT(*) FILTER (WHERE lower(COALESCE(properties::jsonb ->> '$ai_is_error', properties::jsonb ->> 'status', '')) IN ('true', 'error'))::float / COUNT(*) END AS rate FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "rate",
+        "yFormatter": "percent",
+        "description": "Share of tracked LLM generations that ended in an error status."
+      }
+    },
+    {
+      "id": "llm-cost-over-time",
+      "title": "LLM Cost Over Time by App",
+      "chartType": "area",
+      "source": "first-party",
+      "width": 2,
+      "sql": "SELECT event_date AS date, COALESCE(NULLIF(app, ''), NULLIF(template, ''), 'unknown') AS app, COALESCE(SUM(COALESCE(NULLIF(properties::jsonb ->> '$ai_total_cost_usd', '')::float, NULLIF(properties::jsonb ->> 'cost_usd', '')::float, COALESCE(NULLIF(properties::jsonb ->> 'cost_cents_x100', '')::float, 0) / 10000.0)), 0) AS cost_usd FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY event_date, COALESCE(NULLIF(app, ''), NULLIF(template, ''), 'unknown') ORDER BY date, app",
+      "config": {
+        "xKey": "date",
+        "yKey": "cost_usd",
+        "yFormatter": "currency",
+        "pivot": {
+          "xKey": "date",
+          "seriesKey": "app",
+          "valueKey": "cost_usd"
+        },
+        "stacked": true,
+        "description": "Daily estimated LLM cost stacked by emitting app/template."
+      }
+    },
+    {
+      "id": "llm-cost-by-model",
+      "title": "LLM Cost by Model",
+      "chartType": "bar",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, COALESCE(SUM(COALESCE(NULLIF(properties::jsonb ->> '$ai_total_cost_usd', '')::float, NULLIF(properties::jsonb ->> 'cost_usd', '')::float, COALESCE(NULLIF(properties::jsonb ->> 'cost_cents_x100', '')::float, 0) / 10000.0)), 0) AS cost_usd FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') ORDER BY cost_usd DESC LIMIT 20",
+      "config": {
+        "xKey": "model",
+        "yKey": "cost_usd",
+        "yFormatter": "currency",
+        "color": "var(--brand-purple)",
+        "description": "Estimated LLM cost grouped by model over the last 30 days."
+      }
+    },
+    {
+      "id": "llm-tokens-over-time",
+      "title": "LLM Tokens Over Time by Model",
+      "chartType": "area",
+      "source": "first-party",
+      "width": 2,
+      "sql": "SELECT event_date AS date, COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, SUM(COALESCE(NULLIF(properties::jsonb ->> '$ai_input_tokens', '')::float, NULLIF(properties::jsonb ->> 'input_tokens', '')::float, 0) + COALESCE(NULLIF(properties::jsonb ->> '$ai_output_tokens', '')::float, NULLIF(properties::jsonb ->> 'output_tokens', '')::float, 0)) AS tokens FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY event_date, COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') ORDER BY date, model",
+      "config": {
+        "xKey": "date",
+        "yKey": "tokens",
+        "yFormatter": "number",
+        "pivot": {
+          "xKey": "date",
+          "seriesKey": "model",
+          "valueKey": "tokens"
+        },
+        "stacked": true,
+        "description": "Input plus output tokens per day, stacked by model."
+      }
+    },
+    {
+      "id": "llm-latency-by-model",
+      "title": "LLM Latency by Model",
+      "chartType": "bar",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, COALESCE(AVG(COALESCE(NULLIF(properties::jsonb ->> 'duration_ms', '')::float, NULLIF(properties::jsonb ->> '$ai_latency', '')::float * 1000.0)), 0) AS latency_ms FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') ORDER BY latency_ms DESC LIMIT 20",
+      "config": {
+        "xKey": "model",
+        "yKey": "latency_ms",
+        "yFormatter": "duration",
+        "color": "var(--brand-teal)",
+        "description": "Average tracked LLM generation latency grouped by model."
+      }
+    },
+    {
+      "id": "llm-feedback-responses-30d",
+      "title": "LLM Feedback Responses (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT COUNT(*) AS responses FROM analytics_events WHERE event_name = '$ai_feedback' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "responses",
+        "yFormatter": "number",
+        "description": "Explicit thumbs-up and thumbs-down feedback submitted for agent responses over the last 30 days."
+      }
+    },
+    {
+      "id": "llm-positive-feedback-rate-30d",
+      "title": "Positive LLM Feedback (30d)",
+      "chartType": "metric",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT CASE WHEN COUNT(*) = 0 THEN 0 ELSE COUNT(*) FILTER (WHERE lower(properties::jsonb ->> 'sentiment') = 'positive')::float / COUNT(*) END AS rate FROM analytics_events WHERE event_name = '$ai_feedback' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')",
+      "config": {
+        "yKey": "rate",
+        "yFormatter": "percent",
+        "description": "Share of explicit feedback that was thumbs-up over the last 30 days. This is user-provided feedback, not inferred sentiment."
+      }
+    },
+    {
+      "id": "llm-feedback-over-time",
+      "title": "LLM Feedback Sentiment Over Time",
+      "chartType": "area",
+      "source": "first-party",
+      "width": 2,
+      "sql": "SELECT event_date AS date, lower(properties::jsonb ->> 'sentiment') AS sentiment, COUNT(*) AS responses FROM analytics_events WHERE event_name = '$ai_feedback' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY event_date, lower(properties::jsonb ->> 'sentiment') ORDER BY date, CASE WHEN lower(properties::jsonb ->> 'sentiment') = 'positive' THEN 0 ELSE 1 END",
+      "config": {
+        "xKey": "date",
+        "yKey": "responses",
+        "yFormatter": "number",
+        "pivot": {
+          "xKey": "date",
+          "seriesKey": "sentiment",
+          "valueKey": "responses"
+        },
+        "stacked": true,
+        "colors": ["#10b981", "#ef4444"],
+        "description": "Daily explicit thumbs-up and thumbs-down responses over the last 30 days."
+      }
+    },
+    {
+      "id": "llm-feedback-by-model",
+      "title": "LLM Feedback Sentiment by Model",
+      "chartType": "bar",
+      "source": "first-party",
+      "width": 3,
+      "sql": "WITH feedback AS (SELECT COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, lower(properties::jsonb ->> 'sentiment') AS sentiment FROM analytics_events WHERE event_name = '$ai_feedback' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')), top_models AS (SELECT model, COUNT(*) AS total_responses FROM feedback GROUP BY model ORDER BY total_responses DESC LIMIT 20) SELECT feedback.model, feedback.sentiment, COUNT(*) AS responses FROM feedback JOIN top_models ON top_models.model = feedback.model GROUP BY feedback.model, feedback.sentiment, top_models.total_responses ORDER BY top_models.total_responses DESC, feedback.model, CASE WHEN feedback.sentiment = 'positive' THEN 0 ELSE 1 END",
+      "config": {
+        "xKey": "model",
+        "yKey": "responses",
+        "yFormatter": "number",
+        "pivot": {
+          "xKey": "model",
+          "seriesKey": "sentiment",
+          "valueKey": "responses"
+        },
+        "stacked": true,
+        "colors": ["#10b981", "#ef4444"],
+        "description": "Explicit thumbs-up and thumbs-down responses grouped by the model that generated the rated response."
+      }
+    },
+    {
+      "id": "llm-inferred-sentiment-30d",
+      "title": "Inferred Message Sentiment (30d)",
+      "chartType": "bar",
+      "source": "first-party",
+      "width": 1,
+      "sql": "SELECT lower(properties::jsonb ->> 'sentiment') AS sentiment, COUNT(*) AS messages FROM analytics_events WHERE event_name = '$ai_sentiment' AND lower(COALESCE(properties::jsonb ->> 'method', '')) = 'llm' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'neutral', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY lower(properties::jsonb ->> 'sentiment') ORDER BY CASE lower(properties::jsonb ->> 'sentiment') WHEN 'positive' THEN 0 WHEN 'neutral' THEN 1 ELSE 2 END",
+      "config": {
+        "xKey": "sentiment",
+        "yKey": "messages",
+        "yFormatter": "number",
+        "colors": ["#10b981", "#94a3b8", "#ef4444"],
+        "description": "Optional LLM-inferred positive, neutral, and negative message classifications over the last 30 days. This is separate from explicit thumbs feedback."
+      }
+    },
+    {
+      "id": "llm-inferred-sentiment-over-time",
+      "title": "Inferred Message Sentiment Over Time",
+      "chartType": "area",
+      "source": "first-party",
+      "width": 2,
+      "sql": "SELECT event_date AS date, lower(properties::jsonb ->> 'sentiment') AS sentiment, COUNT(*) AS messages FROM analytics_events WHERE event_name = '$ai_sentiment' AND lower(COALESCE(properties::jsonb ->> 'method', '')) = 'llm' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'neutral', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') GROUP BY event_date, lower(properties::jsonb ->> 'sentiment') ORDER BY date, CASE lower(properties::jsonb ->> 'sentiment') WHEN 'positive' THEN 0 WHEN 'neutral' THEN 1 ELSE 2 END",
+      "config": {
+        "xKey": "date",
+        "yKey": "messages",
+        "yFormatter": "number",
+        "pivot": {
+          "xKey": "date",
+          "seriesKey": "sentiment",
+          "valueKey": "messages"
+        },
+        "stacked": true,
+        "colors": ["#10b981", "#94a3b8", "#ef4444"],
+        "description": "Daily optional LLM-inferred message sentiment over the last 30 days, kept separate from explicit thumbs feedback."
+      }
+    },
+    {
+      "id": "llm-inferred-sentiment-by-model",
+      "title": "Inferred Message Sentiment by Main Model",
+      "chartType": "bar",
+      "source": "first-party",
+      "width": 3,
+      "sql": "WITH inferred AS (SELECT COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, lower(properties::jsonb ->> 'sentiment') AS sentiment FROM analytics_events WHERE event_name = '$ai_sentiment' AND lower(COALESCE(properties::jsonb ->> 'method', '')) = 'llm' AND lower(COALESCE(properties::jsonb ->> 'sentiment', '')) IN ('positive', 'neutral', 'negative') AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD')), top_models AS (SELECT model, COUNT(*) AS total_messages FROM inferred GROUP BY model ORDER BY total_messages DESC LIMIT 20) SELECT inferred.model, inferred.sentiment, COUNT(*) AS messages FROM inferred JOIN top_models ON top_models.model = inferred.model GROUP BY inferred.model, inferred.sentiment, top_models.total_messages ORDER BY top_models.total_messages DESC, inferred.model, CASE inferred.sentiment WHEN 'positive' THEN 0 WHEN 'neutral' THEN 1 ELSE 2 END",
+      "config": {
+        "xKey": "model",
+        "yKey": "messages",
+        "yFormatter": "number",
+        "pivot": {
+          "xKey": "model",
+          "seriesKey": "sentiment",
+          "valueKey": "messages"
+        },
+        "stacked": true,
+        "colors": ["#10b981", "#94a3b8", "#ef4444"],
+        "description": "Optional inferred message sentiment grouped by the main model whose response preceded the classified user message, not by the classifier model."
+      }
+    },
+    {
+      "id": "top-expensive-llm-runs",
+      "title": "Top Expensive LLM Runs",
+      "chartType": "table",
+      "source": "first-party",
+      "width": 3,
+      "sql": "SELECT timestamp, COALESCE(NULLIF(app, ''), NULLIF(template, ''), 'unknown') AS app, COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, COALESCE(NULLIF(user_id, ''), NULLIF(anonymous_id, ''), 'anonymous') AS user_key, COALESCE(NULLIF(properties::jsonb ->> 'run_id', ''), NULLIF(properties::jsonb ->> '$ai_trace_id', '')) AS run_id, ROUND((COALESCE(NULLIF(properties::jsonb ->> '$ai_total_cost_usd', '')::float, NULLIF(properties::jsonb ->> 'cost_usd', '')::float, COALESCE(NULLIF(properties::jsonb ->> 'cost_cents_x100', '')::float, 0) / 10000.0))::numeric, 4) AS cost_usd, ROUND((COALESCE(NULLIF(properties::jsonb ->> 'duration_ms', '')::float, NULLIF(properties::jsonb ->> '$ai_latency', '')::float * 1000.0, 0))::numeric) AS duration_ms, COALESCE(NULLIF(properties::jsonb ->> 'tool_calls', '')::int, 0) AS tool_calls FROM analytics_events WHERE event_name = '$ai_generation' AND event_date >= to_char(CURRENT_DATE - INTERVAL '30 days', 'YYYY-MM-DD') ORDER BY cost_usd DESC LIMIT 25",
+      "config": {
+        "description": "Highest estimated-cost agent runs in the last 30 days.",
+        "columns": [
+          {
+            "key": "timestamp",
+            "label": "Time",
+            "format": "date"
+          },
+          {
+            "key": "app",
+            "label": "App"
+          },
+          {
+            "key": "model",
+            "label": "Model"
+          },
+          {
+            "key": "user_key",
+            "label": "User"
+          },
+          {
+            "key": "run_id",
+            "label": "Run ID"
+          },
+          {
+            "key": "cost_usd",
+            "label": "Cost",
+            "format": "currency"
+          },
+          {
+            "key": "duration_ms",
+            "label": "Duration",
+            "format": "duration"
+          },
+          {
+            "key": "tool_calls",
+            "label": "Tools",
+            "format": "number"
+          }
+        ],
+        "sortable": true,
+        "limit": 25
+      }
+    },
+    {
+      "id": "recent-llm-errors",
+      "title": "Recent LLM Errors",
+      "chartType": "table",
+      "source": "first-party",
+      "width": 3,
+      "sql": "SELECT timestamp, COALESCE(NULLIF(app, ''), NULLIF(template, ''), 'unknown') AS app, COALESCE(NULLIF(properties::jsonb ->> '$ai_model', ''), NULLIF(properties::jsonb ->> 'model', ''), 'unknown') AS model, COALESCE(NULLIF(user_id, ''), NULLIF(anonymous_id, ''), 'anonymous') AS user_key, COALESCE(NULLIF(properties::jsonb ->> 'run_id', ''), NULLIF(properties::jsonb ->> '$ai_trace_id', '')) AS run_id, COALESCE(NULLIF(properties::jsonb ->> 'error_message', ''), NULLIF(properties::jsonb ->> '$ai_error', ''), 'error') AS error FROM analytics_events WHERE event_name = '$ai_generation' AND lower(COALESCE(properties::jsonb ->> '$ai_is_error', properties::jsonb ->> 'status', '')) IN ('true', 'error') ORDER BY timestamp DESC LIMIT 25",
+      "config": {
+        "description": "Most recent tracked LLM generation errors.",
+        "columns": [
+          {
+            "key": "timestamp",
+            "label": "Time",
+            "format": "date"
+          },
+          {
+            "key": "app",
+            "label": "App"
+          },
+          {
+            "key": "model",
+            "label": "Model"
+          },
+          {
+            "key": "user_key",
+            "label": "User"
+          },
+          {
+            "key": "run_id",
+            "label": "Run ID"
+          },
+          {
+            "key": "error",
+            "label": "Error"
+          }
+        ],
+        "sortable": true,
+        "limit": 25
+      }
     }
   ]
 }

package/corpus/templates/analytics/server/lib/dashboard-catalog.ts

@@ -938,35 +938,12 @@ export const dashboardCatalogEntries: DashboardCatalogEntry[] = [
       "referrals",
       "virality",
     ],
-    panelCount: 34,
+    panelCount: 51,
     version: CATALOG_VERSION,
     recommended: true,
     visibleInCatalog: false,
     buildConfig: () => seedConfig("agent-native-templates-first-party"),
   },
-  {
-    id: "agent-observability-llm",
-    name: "Agent LLM Observability",
-    description:
-      "LLM generation volume, latency, errors, token usage, and cost captured as first-party `$ai_generation` tracking events.",
-    category: "Observability",
-    defaultDashboardId: "agent-observability-llm",
-    dataSources: ["first-party"],
-    tags: [
-      "ai",
-      "llm",
-      "observability",
-      "cost",
-      "tokens",
-      "latency",
-      "posthog",
-      "first-party",
-    ],
-    panelCount: 10,
-    version: CATALOG_VERSION,
-    recommended: true,
-    buildConfig: () => seedConfig("agent-observability-llm"),
-  },
   {
     id: "skills-cli-funnel",
     name: "Skills CLI Funnel",

package/corpus/templates/analytics/server/lib/dashboard-seeds.ts

@@ -3,16 +3,11 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 
 import firstPartyTemplateTrafficSeed from "../../seeds/dashboards/agent-native-templates-first-party.json" with { type: "json" };
-import agentObservabilityLlmSeed from "../../seeds/dashboards/agent-observability-llm.json" with { type: "json" };
 import googleAnalyticsSeed from "../../seeds/dashboards/google-analytics.json" with { type: "json" };
 import nodeExporterFullSeed from "../../seeds/dashboards/node-exporter-full.json" with { type: "json" };
 import skillsCliFunnelSeed from "../../seeds/dashboards/skills-cli-funnel.json" with { type: "json" };
 
 const shippedSeeds: Record<string, Record<string, unknown>> = {
-  "agent-observability-llm": agentObservabilityLlmSeed as Record<
-    string,
-    unknown
-  >,
   "agent-native-templates-first-party": firstPartyTemplateTrafficSeed as Record<
     string,
     unknown