@agent-native/core 0.141.7 → 0.144.1

package/corpus/templates/analytics/server/lib/first-party-analytics-backend.ts

@@ -73,6 +73,8 @@ const FIRST_PARTY_QUERY_TABLES = [
   "analytics_user_days",
 ] as const;
 const BACKEND_CONFIG_CACHE_TTL_MS = 30_000;
+const MAX_BACKFILL_BATCH_SIZE = 5_000;
+const MAX_INSERT_BATCH_SIZE = 200;
 
 const backendConfigCache = new Map<
   string,
@@ -222,6 +224,30 @@ const FIRST_PARTY_ANALYTICS_RAW_SCHEMA = [
   ["org_id", "STRING"],
 ] as const;
 
+const FIRST_PARTY_ANALYTICS_BACKFILL_COLUMNS = [
+  "id",
+  "public_key_id",
+  "event_name",
+  "user_id",
+  "anonymous_id",
+  "user_key",
+  "session_id",
+  "timestamp",
+  "event_date",
+  "received_at",
+  "url",
+  "path",
+  "hostname",
+  "referrer",
+  "app",
+  "template",
+  "signed_in",
+  "properties",
+  "context",
+  "owner_email",
+  "org_id",
+] as const;
+
 const FIRST_PARTY_ANALYTICS_QUERY_SCHEMA = FIRST_PARTY_ANALYTICS_RAW_SCHEMA;
 const FIRST_PARTY_ANALYTICS_DAILY_ROLLUP_SCHEMA = [
   ["id", "STRING"],
@@ -337,8 +363,16 @@ export async function insertFirstPartyAnalyticsRows(
     getAccessToken(),
   ]);
   const payloadRows = rows.map(firstPartyEventRowToBigQuery);
-  for (let offset = 0; offset < payloadRows.length; offset += 50) {
-    await insertBatch(table, token, payloadRows.slice(offset, offset + 50));
+  for (
+    let offset = 0;
+    offset < payloadRows.length;
+    offset += MAX_INSERT_BATCH_SIZE
+  ) {
+    await insertBatch(
+      table,
+      token,
+      payloadRows.slice(offset, offset + MAX_INSERT_BATCH_SIZE),
+    );
   }
   return payloadRows.length;
 }
@@ -982,77 +1016,154 @@ function serializeBackfillCursor(
   return JSON.stringify(cursor);
 }
 
-function backfillScopeSql(
-  scope: FirstPartyAnalyticsScope,
+function backfillBranchSql(
+  predicate: string,
+  predicateArgs: string[],
   cursor: FirstPartyAnalyticsBackfillCursor,
 ): {
   sql: string;
   args: string[];
 } {
-  const cursorSql =
-    "(received_at > ? OR (received_at = ? AND id > ?)) ORDER BY received_at ASC, id ASC LIMIT ?";
-  if (scope.orgId) {
-    return {
-      sql:
-        "WHERE (org_id = ? OR (org_id IS NULL AND owner_email = ?)) AND " +
-        cursorSql,
-      args: [
-        scope.orgId,
-        scope.userEmail,
-        cursor.receivedAt,
-        cursor.receivedAt,
-        cursor.id,
-      ],
-    };
-  }
+  const cursorSql = cursor.receivedAt
+    ? "(received_at > ? OR (received_at = ? AND id > ?))"
+    : "";
+  const cursorArgs = cursor.receivedAt
+    ? [cursor.receivedAt, cursor.receivedAt, cursor.id]
+    : [];
   return {
-    sql: "WHERE org_id IS NULL AND owner_email = ? AND " + cursorSql,
-    args: [scope.userEmail, cursor.receivedAt, cursor.receivedAt, cursor.id],
+    sql: `SELECT id, received_at
+      FROM analytics_events
+      WHERE ${predicate}${cursorSql ? ` AND ${cursorSql}` : ""}
+      ORDER BY received_at ASC, id ASC LIMIT ?`,
+    args: [...predicateArgs, ...cursorArgs],
   };
 }
 
+function backfillRowsByIdsSql(ids: string[]): {
+  sql: string;
+  args: string[];
+} {
+  return {
+    sql: `SELECT ${FIRST_PARTY_ANALYTICS_BACKFILL_COLUMNS.join(", ")}
+      FROM analytics_events
+      WHERE id IN (${ids.map(() => "?").join(", ")})`,
+    args: ids,
+  };
+}
+
+const MAX_SQLITE_BIND_VARIABLES = 900;
+
+function backfillRowCursor(
+  row: Record<string, unknown>,
+): FirstPartyAnalyticsBackfillCursor {
+  const id = row.id;
+  const receivedAt = row.received_at ?? row.receivedAt;
+  if (typeof id !== "string" || !id) {
+    throw new Error("First-party analytics backfill row is missing its id");
+  }
+  if (typeof receivedAt !== "string" || !receivedAt) {
+    throw new Error(
+      "First-party analytics backfill row is missing its received_at",
+    );
+  }
+  return { receivedAt, id };
+}
+
+function compareBackfillRows(
+  left: Record<string, unknown>,
+  right: Record<string, unknown>,
+): number {
+  const leftCursor = backfillRowCursor(left);
+  const rightCursor = backfillRowCursor(right);
+  return (
+    leftCursor.receivedAt.localeCompare(rightCursor.receivedAt) ||
+    leftCursor.id.localeCompare(rightCursor.id)
+  );
+}
+
 export async function backfillFirstPartyAnalyticsBatch(
   scope: FirstPartyAnalyticsScope,
   cursor: string | null,
   limit: number,
   configuredTable?: string | null,
 ): Promise<FirstPartyAnalyticsBackfillBatch> {
-  const boundedLimit = Math.min(Math.max(Math.floor(limit), 1), 500);
+  const boundedLimit = Math.min(
+    Math.max(Math.floor(limit), 1),
+    MAX_BACKFILL_BATCH_SIZE,
+  );
   const db = getDbExec();
   const parsedCursor = parseBackfillCursor(cursor);
-  const scoped = backfillScopeSql(scope, parsedCursor);
-  const result = await db.execute({
-    sql: `SELECT * FROM analytics_events ${scoped.sql}`,
-    args: [...scoped.args, boundedLimit],
-    timeoutMs: 20_000,
-    maxAttempts: 1,
-  });
-  const rows = result.rows as Record<string, unknown>[];
-  if (!rows.length) {
+  const branches = scope.orgId
+    ? [
+        { predicate: "org_id = ?", args: [scope.orgId] },
+        {
+          predicate: "org_id IS NULL AND owner_email = ?",
+          args: [scope.userEmail],
+        },
+      ]
+    : [
+        {
+          predicate: "org_id IS NULL AND owner_email = ?",
+          args: [scope.userEmail],
+        },
+      ];
+  const rows: Record<string, unknown>[] = [];
+  for (const branch of branches) {
+    const scoped = backfillBranchSql(
+      branch.predicate,
+      branch.args,
+      parsedCursor,
+    );
+    const result = await db.execute({
+      sql: scoped.sql,
+      args: [...scoped.args, boundedLimit],
+      timeoutMs: 20_000,
+      maxAttempts: 1,
+    });
+    rows.push(...(result.rows as Record<string, unknown>[]));
+  }
+  rows.sort(compareBackfillRows);
+  const selectedRows = rows.slice(0, boundedLimit);
+  if (!selectedRows.length) {
     return {
       nextCursor: serializeBackfillCursor(parsedCursor),
       copied: 0,
       complete: true,
     };
   }
-  await insertFirstPartyAnalyticsRows(rows, configuredTable);
-  const lastId = rows[rows.length - 1]?.id;
-  if (typeof lastId !== "string" || !lastId) {
-    throw new Error("First-party analytics backfill row is missing its id");
-  }
-  const lastReceivedAt =
-    rows[rows.length - 1]?.received_at ?? rows[rows.length - 1]?.receivedAt;
-  if (typeof lastReceivedAt !== "string" || !lastReceivedAt) {
-    throw new Error(
-      "First-party analytics backfill row is missing its received_at",
+
+  const selectedIds = selectedRows.map((row) => backfillRowCursor(row).id);
+  const hydratedRows: Record<string, unknown>[] = [];
+  for (
+    let offset = 0;
+    offset < selectedIds.length;
+    offset += MAX_SQLITE_BIND_VARIABLES
+  ) {
+    const hydratedQuery = backfillRowsByIdsSql(
+      selectedIds.slice(offset, offset + MAX_SQLITE_BIND_VARIABLES),
     );
+    const hydratedResult = await db.execute({
+      sql: hydratedQuery.sql,
+      args: hydratedQuery.args,
+      timeoutMs: 20_000,
+      maxAttempts: 1,
+    });
+    hydratedRows.push(...(hydratedResult.rows as Record<string, unknown>[]));
   }
+  const hydratedById = new Map(
+    hydratedRows.map((row) => [backfillRowCursor(row).id, row]),
+  );
+  const selectedEvents = selectedIds
+    .map((id) => hydratedById.get(id))
+    .filter((row): row is Record<string, unknown> => row !== undefined);
+  await insertFirstPartyAnalyticsRows(selectedEvents, configuredTable);
+  const lastCursor = backfillRowCursor(selectedRows[selectedRows.length - 1]!);
   return {
     nextCursor: serializeBackfillCursor({
-      receivedAt: lastReceivedAt,
-      id: lastId,
+      receivedAt: lastCursor.receivedAt,
+      id: lastCursor.id,
     }),
-    copied: rows.length,
+    copied: selectedEvents.length,
     complete: rows.length < boundedLimit,
   };
 }

package/corpus/templates/analytics/server/lib/native-v2-dashboards.ts

@@ -10,6 +10,7 @@ export const NATIVE_V2_DASHBOARD_IDS = [
   "account-engagement-v2",
   "cross-sell-v2",
   "win-loss-v2",
+  "on-demand-billing-v2",
 ] as const;
 
 export type NativeV2DashboardId = (typeof NATIVE_V2_DASHBOARD_IDS)[number];
@@ -29,7 +30,16 @@ export type NativeV2BindingKey =
   | "win_loss.summary"
   | "win_loss.reasons"
   | "win_loss.evidence"
-  | "win_loss.alerts";
+  | "win_loss.alerts"
+  | "billing.visual_views.summary"
+  | "billing.visual_views.trend"
+  | "billing.visual_views.overage"
+  | "billing.visual_views.growth"
+  | "billing.visual_views.insights"
+  | "billing.agent_credits.summary"
+  | "billing.agent_credits.trend"
+  | "billing.agent_credits.overage"
+  | "billing.agent_credits.insights";
 
 export interface NativeV2Binding {
   programId: string;
@@ -387,6 +397,222 @@ export const nativeV2DashboardManifests: readonly NativeV2DashboardManifest[] =
         }),
       ],
     },
+    {
+      id: "on-demand-billing-v2",
+      name: "On Demand Billing v2",
+      description:
+        "Native on-demand billing dashboard for Visual Views and Agent Credits: usage, rollover, overage, growth signals, and billing insights.",
+      category: "Operations",
+      tags: [
+        "native",
+        "v2",
+        "billing",
+        "on-demand",
+        "visual-views",
+        "agent-credits",
+      ],
+      requiredBindings: [
+        "billing.visual_views.summary",
+        "billing.visual_views.trend",
+        "billing.visual_views.overage",
+        "billing.visual_views.growth",
+        "billing.visual_views.insights",
+        "billing.agent_credits.summary",
+        "billing.agent_credits.trend",
+        "billing.agent_credits.overage",
+        "billing.agent_credits.insights",
+      ],
+      panels: [
+        section(
+          "billing-visual-views-overview",
+          "Visual Views",
+          "Visual Views / Overview",
+        ),
+        boundPanel({
+          id: "billing-visual-views-summary",
+          title: "Visual Views summary",
+          bindingKey: "billing.visual_views.summary",
+          chartType: "metric",
+          width: 1,
+          tab: "Visual Views / Overview",
+          config: {
+            yKey: "value",
+            yFormatter: "number",
+            description:
+              "Current Visual Views usage or contracted allowance, as emitted by the bound program.",
+          },
+          requiredColumns: ["value"],
+        }),
+        boundPanel({
+          id: "billing-visual-views-trend",
+          title: "Usage over time",
+          bindingKey: "billing.visual_views.trend",
+          chartType: "area",
+          width: 3,
+          tab: "Visual Views / Overview",
+          config: {
+            xKey: "month",
+            yKeys: ["actual", "contracted", "rollover"],
+            yFormatter: "number",
+            legend: true,
+          },
+          requiredColumns: ["month", "actual", "contracted", "rollover"],
+        }),
+        boundPanel({
+          id: "billing-visual-views-insights",
+          title: "Billing insights",
+          bindingKey: "billing.visual_views.insights",
+          chartType: "callout",
+          width: 3,
+          tab: "Visual Views / Overview",
+          requiredColumns: ["severity", "message"],
+        }),
+        boundPanel({
+          id: "billing-visual-views-overage",
+          title: "Overage and rollover",
+          bindingKey: "billing.visual_views.overage",
+          chartType: "table",
+          width: 3,
+          tab: "Visual Views / Overage",
+          config: {
+            columns: [
+              { key: "company_name", label: "Company" },
+              { key: "month", label: "Month", format: "date" },
+              { key: "actual", label: "Actual", format: "number" },
+              { key: "contracted", label: "Contracted", format: "number" },
+              { key: "rollover", label: "Rollover", format: "number" },
+              {
+                key: "newAllowance",
+                label: "New allowance",
+                format: "number",
+              },
+              { key: "overage", label: "Overage", format: "number" },
+              {
+                key: "overageRate",
+                label: "Overage rate",
+                format: "currency",
+              },
+              {
+                key: "overageBilled",
+                label: "Overage billed",
+                format: "currency",
+              },
+            ],
+            limit: 100,
+          },
+          requiredColumns: [
+            "company_name",
+            "month",
+            "actual",
+            "contracted",
+            "rollover",
+            "newAllowance",
+            "overage",
+            "overageRate",
+            "overageBilled",
+          ],
+        }),
+        boundPanel({
+          id: "billing-visual-views-growth",
+          title: "High growth companies",
+          bindingKey: "billing.visual_views.growth",
+          chartType: "table",
+          width: 3,
+          tab: "Visual Views / Growth",
+          config: { limit: 100 },
+          requiredColumns: [],
+        }),
+        section(
+          "billing-agent-credits-overview",
+          "Agent Credits",
+          "Agent Credits / Overview",
+        ),
+        boundPanel({
+          id: "billing-agent-credits-summary",
+          title: "Agent Credits summary",
+          bindingKey: "billing.agent_credits.summary",
+          chartType: "metric",
+          width: 1,
+          tab: "Agent Credits / Overview",
+          config: {
+            yKey: "value",
+            yFormatter: "number",
+            description:
+              "Current Agent Credits usage or contracted allowance, as emitted by the bound program.",
+          },
+          requiredColumns: ["value"],
+        }),
+        boundPanel({
+          id: "billing-agent-credits-trend",
+          title: "Credits over time",
+          bindingKey: "billing.agent_credits.trend",
+          chartType: "area",
+          width: 3,
+          tab: "Agent Credits / Overview",
+          config: {
+            xKey: "month",
+            yKeys: ["actual", "contracted", "rollover"],
+            yFormatter: "number",
+            legend: true,
+          },
+          requiredColumns: ["month", "actual", "contracted", "rollover"],
+        }),
+        boundPanel({
+          id: "billing-agent-credits-insights",
+          title: "Billing insights",
+          bindingKey: "billing.agent_credits.insights",
+          chartType: "callout",
+          width: 3,
+          tab: "Agent Credits / Overview",
+          requiredColumns: ["severity", "message"],
+        }),
+        boundPanel({
+          id: "billing-agent-credits-overage",
+          title: "Overage and rollover",
+          bindingKey: "billing.agent_credits.overage",
+          chartType: "table",
+          width: 3,
+          tab: "Agent Credits / Overage",
+          config: {
+            columns: [
+              { key: "company_name", label: "Company" },
+              { key: "month", label: "Month", format: "date" },
+              { key: "actual", label: "Actual", format: "number" },
+              { key: "contracted", label: "Contracted", format: "number" },
+              { key: "rollover", label: "Rollover", format: "number" },
+              {
+                key: "newAllowance",
+                label: "New allowance",
+                format: "number",
+              },
+              { key: "overage", label: "Overage", format: "number" },
+              {
+                key: "overageRate",
+                label: "Overage rate",
+                format: "currency",
+              },
+              {
+                key: "overageBilled",
+                label: "Overage billed",
+                format: "currency",
+              },
+            ],
+            limit: 100,
+          },
+          requiredColumns: [
+            "company_name",
+            "month",
+            "actual",
+            "contracted",
+            "rollover",
+            "newAllowance",
+            "overage",
+            "overageRate",
+            "overageBilled",
+          ],
+        }),
+      ],
+    },
   ] as const;
 
 export function getNativeV2DashboardManifest(

package/corpus/templates/analytics/server/plugins/agent-chat.ts

@@ -10,6 +10,7 @@ import actionsRegistry from "../../.generated/actions-registry.js";
 import { INITIAL_TOOL_NAMES } from "../lib/agent-chat-plan-mode";
 import { ANALYTICS_CONNECTOR_CATALOG } from "../lib/analytics-connector-catalog";
 import { credentialProviderConfigs } from "../lib/credential-keys";
+import { isProductionServerlessRuntime } from "../lib/production-serverless-runtime.js";
 import {
   draftClaimsAnalyticsMetrics,
   failedDataQueryAttemptMessage,
@@ -881,6 +882,10 @@ export async function searchDashboardMentions(query: string, event?: any) {
 
 export default createAgentChatPlugin({
   appId: "analytics",
+  // Resource prompt hydration performs additive schema checks. Keep that
+  // work out of production serverless cold starts; it is not needed for the
+  // dashboard's domain prompt and can contend with the request's DB queries.
+  leanPrompt: isProductionServerlessRuntime(),
   actions: loadActionsFromStaticRegistry(actionsRegistry),
   initialToolNames: INITIAL_TOOL_NAMES,
   corpusTools: "lazy",

package/corpus/templates/analytics/server/plugins/analytics-rollup-backfill-jobs.ts

@@ -13,6 +13,7 @@ function platformSchedulerOwnsBackfill(): boolean {
   return (
     process.env.NETLIFY === "true" ||
     Boolean(process.env.NETLIFY_FUNCTION_NAME) ||
+    process.env.NITRO_PRESET === "netlify" ||
     globalThis.__AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__ ===
       true
   );

package/corpus/templates/analytics/server/plugins/db.ts

@@ -1,5 +1,4 @@
 import {
-  deferMigration,
   ensureAdditiveColumns,
   getDbExec,
   runMigrations,
@@ -10,8 +9,7 @@ import { isInBackgroundFunctionRuntime } from "@agent-native/core/server";
 // startup so the dashboard / analysis share actions know where to dispatch.
 import "../db/index.js";
 import * as schema from "../db/schema.js";
-import { isHistoricalAnalyticsRollupBackfillComplete } from "../jobs/analytics-rollup-backfill.js";
-import { repairPersistedFirstPartyDashboardQueries } from "../lib/first-party-dashboard-repair.js";
+import { isProductionServerlessRuntime } from "../lib/production-serverless-runtime.js";
 
 /**
  * Every Drizzle table exported from schema.ts. Filters out type-only and
@@ -1407,11 +1405,9 @@ const runAnalyticsMigrations = runMigrations(
       version: 134,
       name: "analytics-rollups-historical-backfill-repair",
       sql: {},
-      run: async () => {
-        if (!(await isHistoricalAnalyticsRollupBackfillComplete())) {
-          return deferMigration();
-        }
-      },
+      // The historical rebuild is an out-of-band job. Do not defer this
+      // marker until it completes: a pending named migration is retried by
+      // every cold start and turns a recoverable backfill into a boot blocker.
     },
     {
       version: 135,
@@ -1421,6 +1417,28 @@ const runAnalyticsMigrations = runMigrations(
         ALTER TABLE analytics_rollup_backfill_state ADD COLUMN IF NOT EXISTS lease_expires_at TEXT;
       `,
     },
+    {
+      version: 136,
+      name: "analytics-events-backfill-cursor-indexes",
+      sql: {
+        postgres: `CREATE INDEX CONCURRENTLY IF NOT EXISTS analytics_events_org_received_id_idx ON analytics_events (org_id, received_at, id); CREATE INDEX CONCURRENTLY IF NOT EXISTS analytics_events_owner_received_id_idx ON analytics_events (owner_email, received_at, id) WHERE org_id IS NULL`,
+        sqlite: `CREATE INDEX IF NOT EXISTS analytics_events_org_received_id_idx ON analytics_events (org_id, received_at, id); CREATE INDEX IF NOT EXISTS analytics_events_owner_received_id_idx ON analytics_events (owner_email, received_at, id) WHERE org_id IS NULL`,
+      },
+    },
+    {
+      version: 137,
+      name: "dashboard-name-locks",
+      sql: {
+        postgres: `CREATE TABLE IF NOT EXISTS dashboard_name_locks (
+          name_key TEXT PRIMARY KEY,
+          created_at TEXT NOT NULL DEFAULT (now()::text)
+        )`,
+        sqlite: `CREATE TABLE IF NOT EXISTS dashboard_name_locks (
+          name_key TEXT PRIMARY KEY,
+          created_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )`,
+      },
+    },
   ],
   { table: "analytics_migrations" },
 );
@@ -1452,18 +1470,34 @@ export default async (nitroApp: any): Promise<void> => {
     );
     return;
   }
+  const isNetlifyServerlessRuntime =
+    isProductionServerlessRuntime() ||
+    process.env.NETLIFY === "true" ||
+    Boolean(process.env.NETLIFY_FUNCTION_NAME) ||
+    Boolean(process.env.AWS_LAMBDA_FUNCTION_NAME) ||
+    Boolean(process.env.LAMBDA_TASK_ROOT);
+  if (
+    isNetlifyServerlessRuntime &&
+    process.env.ANALYTICS_SKIP_BOOT_MIGRATIONS === "1"
+  ) {
+    console.info(
+      "[db] Skipping Analytics migrations in production serverless runtime by explicit incident flag",
+    );
+    return;
+  }
+  // The schema must exist before the first query. Measured cost on this
+  // database (180 tables): ~5.5s for the version check alone, which is why the
+  // serverless runtime skips this entirely via ANALYTICS_SKIP_BOOT_MIGRATIONS
+  // above rather than paying it on every cold start.
+  // guard:allow-boot-data-work — schema must exist before the first query
   await runAnalyticsMigrations(nitroApp);
-  try {
-    if (await repairPersistedFirstPartyDashboardQueries()) {
-      console.info(
-        "[db] Repaired bounded recurring-user queries on the canonical first-party dashboard.",
-      );
-    }
-  } catch (err) {
-    console.warn(
-      "[db] Failed to repair canonical first-party dashboard queries (non-fatal):",
-      err instanceof Error ? err.message : err,
+  if (isNetlifyServerlessRuntime) {
+    // The migration list is authoritative; repeating repair and schema
+    // introspection on every function cold start only adds pool pressure.
+    console.info(
+      "[db] Skipping post-migration schema convergence in production serverless runtime",
     );
+    return;
   }
   try {
     const summary = await ensureAdditiveColumns({