@agent-native/core 0.141.5 → 0.141.7

package/corpus/templates/analytics/server/lib/first-party-analytics-health.ts

@@ -4,6 +4,10 @@ import { and, gte, inArray, lte } from "drizzle-orm";
 
 import { getDb, schema } from "../db/index.js";
 import { hasCredential } from "./credentials.js";
+import {
+  getFirstPartyAnalyticsBackend,
+  getFirstPartyAnalyticsBigQueryMetrics,
+} from "./first-party-analytics-backend.js";
 import type { AnalyticsScope } from "./first-party-analytics.js";
 
 export const FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS = {
@@ -13,11 +17,33 @@ export const FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS = {
   recommendMaxQueryMs: 30_000,
 } as const;
 
-const BIGQUERY_SETUP_LINK = buildDeepLink({
-  app: "analytics",
-  view: "data-sources",
-  to: "/data-sources?source=bigquery",
-});
+const EXTERNAL_BACKEND_DEFINITIONS = [
+  {
+    id: "bigquery",
+    label: "BigQuery",
+    role: "warehouse",
+    requiredKeys: [
+      "GOOGLE_APPLICATION_CREDENTIALS_JSON",
+      "BIGQUERY_PROJECT_ID",
+    ],
+    setupLink: buildDeepLink({
+      app: "analytics",
+      view: "data-sources",
+      to: "/data-sources?source=bigquery",
+    }),
+  },
+  {
+    id: "amplitude",
+    label: "Amplitude",
+    role: "product-analytics",
+    requiredKeys: ["AMPLITUDE_API_KEY", "AMPLITUDE_SECRET_KEY"],
+    setupLink: buildDeepLink({
+      app: "analytics",
+      view: "data-sources",
+      to: "/data-sources?source=amplitude",
+    }),
+  },
+] as const;
 
 export type FirstPartyAnalyticsQueryClass =
   | "raw-events"
@@ -39,9 +65,29 @@ export type FirstPartyAnalyticsRecommendation =
   | "connect_bigquery"
   | "use_bigquery";
 
+export type FirstPartyAnalyticsExternalBackendRecommendation =
+  | "none"
+  | "connect"
+  | "use"
+  | "unknown";
+
+export type FirstPartyAnalyticsBackendId =
+  (typeof EXTERNAL_BACKEND_DEFINITIONS)[number]["id"];
+
+export interface FirstPartyAnalyticsBackendStatus {
+  id: FirstPartyAnalyticsBackendId;
+  label: string;
+  role: (typeof EXTERNAL_BACKEND_DEFINITIONS)[number]["role"];
+  configured: boolean | null;
+  missingRequiredKeys: string[];
+  setupLink: string;
+}
+
 export interface FirstPartyAnalyticsHealth {
   status: FirstPartyAnalyticsHealthStatus;
   recommendation: FirstPartyAnalyticsRecommendation;
+  externalBackendRecommendation: FirstPartyAnalyticsExternalBackendRecommendation;
+  externalBackends: FirstPartyAnalyticsBackendStatus[];
   reasons: Array<"event_volume" | "slow_queries" | "query_timeout">;
   observedAt: string;
   metrics: {
@@ -56,11 +102,8 @@ export interface FirstPartyAnalyticsHealth {
     maxQueryDurationMs24h: number;
   };
   thresholds: typeof FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS;
-  bigQuery: {
-    configured: boolean | null;
-    missingRequiredKeys: string[];
-    setupLink: string;
-  };
+  /** Kept for clients that still read the original BigQuery-only field. */
+  bigQuery: FirstPartyAnalyticsBackendStatus;
 }
 
 interface QueryPressureEvent {
@@ -199,18 +242,13 @@ function dateSpanDays(
   return Math.floor((last - first) / 86_400_000) + 1;
 }
 
-async function bigQueryStatus(scope: AnalyticsScope): Promise<{
-  configured: boolean | null;
-  missingRequiredKeys: string[];
-  setupLink: string;
-}> {
-  const requiredKeys = [
-    "GOOGLE_APPLICATION_CREDENTIALS_JSON",
-    "BIGQUERY_PROJECT_ID",
-  ];
+async function externalBackendStatus(
+  scope: AnalyticsScope,
+  definition: (typeof EXTERNAL_BACKEND_DEFINITIONS)[number],
+): Promise<FirstPartyAnalyticsBackendStatus> {
   try {
     const configured = await Promise.all(
-      requiredKeys.map((key) =>
+      definition.requiredKeys.map((key) =>
         hasCredential(key, {
           userEmail: scope.userEmail,
           orgId: scope.orgId,
@@ -218,22 +256,41 @@ async function bigQueryStatus(scope: AnalyticsScope): Promise<{
       ),
     );
     return {
+      id: definition.id,
+      label: definition.label,
+      role: definition.role,
       configured: configured.every(Boolean),
-      missingRequiredKeys: requiredKeys.filter(
+      missingRequiredKeys: definition.requiredKeys.filter(
         (_, index) => !configured[index],
       ),
-      setupLink: BIGQUERY_SETUP_LINK,
+      setupLink: definition.setupLink,
     };
   } catch (error) {
-    console.warn("[first-party-analytics] BigQuery status unavailable:", error);
+    console.warn(
+      `[first-party-analytics] ${definition.label} status unavailable:`,
+      error,
+    );
     return {
+      id: definition.id,
+      label: definition.label,
+      role: definition.role,
       configured: null,
       missingRequiredKeys: [],
-      setupLink: BIGQUERY_SETUP_LINK,
+      setupLink: definition.setupLink,
     };
   }
 }
 
+async function externalBackendStatuses(
+  scope: AnalyticsScope,
+): Promise<FirstPartyAnalyticsBackendStatus[]> {
+  return Promise.all(
+    EXTERNAL_BACKEND_DEFINITIONS.map((definition) =>
+      externalBackendStatus(scope, definition),
+    ),
+  );
+}
+
 function recommendationFor(
   status: FirstPartyAnalyticsHealthStatus,
   configured: boolean | null,
@@ -242,6 +299,18 @@ function recommendationFor(
   return configured === true ? "use_bigquery" : "connect_bigquery";
 }
 
+function externalBackendRecommendationFor(
+  status: FirstPartyAnalyticsHealthStatus,
+  backends: FirstPartyAnalyticsBackendStatus[],
+): FirstPartyAnalyticsExternalBackendRecommendation {
+  if (status !== "recommend_bigquery") return "none";
+  if (backends.some((backend) => backend.configured === true)) return "use";
+  if (backends.every((backend) => backend.configured === false)) {
+    return "connect";
+  }
+  return "unknown";
+}
+
 export async function getFirstPartyAnalyticsHealth(
   scope: AnalyticsScope,
 ): Promise<FirstPartyAnalyticsHealth> {
@@ -255,19 +324,30 @@ export async function getFirstPartyAnalyticsHealth(
   const rollups = schema.analyticsEventDailyRollups;
   const pressure = schema.analyticsQueryPressureDaily;
   const db = getDb() as any;
+  const backend = await getFirstPartyAnalyticsBackend(scope);
 
-  const [rollupRows, pressureRows, bigQuery] = await Promise.all([
-    db
-      .select({
-        eventCount: sql<number>`coalesce(sum(${rollups.eventCount}), 0)`,
-        dailyRollupRows: sql<number>`count(*)`,
-        firstEventDate: sql<string | null>`min(${rollups.eventDate})`,
-        lastEventDate: sql<string | null>`max(${rollups.eventDate})`,
-      })
-      .from(rollups)
-      .where(
-        and(inArray(rollups.tenantKey, keys), lte(rollups.eventDate, today)),
-      ),
+  const rollupRowsPromise =
+    backend.sink === "bigquery"
+      ? getFirstPartyAnalyticsBigQueryMetrics(scope, backend.table).then(
+          (metrics) => [metrics],
+        )
+      : db
+          .select({
+            eventCount: sql<number>`coalesce(sum(${rollups.eventCount}), 0)`,
+            dailyRollupRows: sql<number>`count(*)`,
+            firstEventDate: sql<string | null>`min(${rollups.eventDate})`,
+            lastEventDate: sql<string | null>`max(${rollups.eventDate})`,
+          })
+          .from(rollups)
+          .where(
+            and(
+              inArray(rollups.tenantKey, keys),
+              lte(rollups.eventDate, today),
+            ),
+          );
+
+  const [rollupRows, pressureRows, externalBackends] = await Promise.all([
+    rollupRowsPromise,
     db
       .select({
         queryClass: pressure.queryClass,
@@ -284,9 +364,14 @@ export async function getFirstPartyAnalyticsHealth(
           lte(pressure.lastSeenAt, nowTimestamp),
         ),
       ),
-    bigQueryStatus(scope),
+    externalBackendStatuses(scope),
   ]);
 
+  const bigQuery = externalBackends.find(
+    (backend) => backend.id === "bigquery",
+  );
+  if (!bigQuery) throw new Error("BigQuery backend status is missing");
+
   const rollup = (rollupRows[0] ?? {}) as Record<string, unknown>;
   const eventCount = finiteNumber(rollup.eventCount);
   const dailyRollupRows = finiteNumber(rollup.dailyRollupRows);
@@ -339,15 +424,24 @@ export async function getFirstPartyAnalyticsHealth(
       FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.recommendEventCount / 4 ||
     slowQueryCount24h > 0;
   const status: FirstPartyAnalyticsHealthStatus =
-    reasons.length > 0
-      ? "recommend_bigquery"
-      : hasMonitorSignal
+    backend.sink === "bigquery"
+      ? slowQueryCount24h > 0 || timeoutCount24h > 0
         ? "monitor"
-        : "healthy";
+        : "healthy"
+      : reasons.length > 0
+        ? "recommend_bigquery"
+        : hasMonitorSignal
+          ? "monitor"
+          : "healthy";
 
   return {
     status,
     recommendation: recommendationFor(status, bigQuery.configured),
+    externalBackendRecommendation: externalBackendRecommendationFor(
+      status,
+      externalBackends,
+    ),
+    externalBackends,
     reasons,
     observedAt: nowIso(),
     metrics: {
@@ -367,9 +461,24 @@ export async function getFirstPartyAnalyticsHealth(
 }
 
 export function unavailableFirstPartyAnalyticsHealth(): FirstPartyAnalyticsHealth {
+  const externalBackends = EXTERNAL_BACKEND_DEFINITIONS.map((definition) => ({
+    id: definition.id,
+    label: definition.label,
+    role: definition.role,
+    configured: null,
+    missingRequiredKeys: [],
+    setupLink: definition.setupLink,
+  }));
+  const bigQuery = externalBackends.find(
+    (backend) => backend.id === "bigquery",
+  );
+  if (!bigQuery) throw new Error("BigQuery backend status is missing");
+
   return {
     status: "unavailable",
     recommendation: "none",
+    externalBackendRecommendation: "unknown",
+    externalBackends,
     reasons: [],
     observedAt: nowIso(),
     metrics: {
@@ -384,10 +493,6 @@ export function unavailableFirstPartyAnalyticsHealth(): FirstPartyAnalyticsHealt
       maxQueryDurationMs24h: 0,
     },
     thresholds: FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS,
-    bigQuery: {
-      configured: null,
-      missingRequiredKeys: [],
-      setupLink: BIGQUERY_SETUP_LINK,
-    },
+    bigQuery,
   };
 }

package/corpus/templates/analytics/server/lib/first-party-analytics-rollups.ts

@@ -1,6 +1,11 @@
+import { isPostgres } from "@agent-native/core/db";
 import { sql } from "@agent-native/core/db/schema";
 
 import { getDb, schema } from "../db/index.js";
+import {
+  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY,
+  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
+} from "./analytics-rollup-lock.js";
 
 /**
  * The normalized subset emitted by first-party analytics ingest. Keeping this
@@ -23,15 +28,6 @@ export interface FirstPartyAnalyticsRollupResult {
   userDayCount: number;
 }
 
-/**
- * Shared with the historical backfill so live ingest and migration work hold
- * the same transaction-scoped Postgres lock while they update rollups.
- */
-export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY =
-  "agent-native:analytics-rollup-backfill";
-export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL =
-  "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))";
-
 interface DailyRollupRow {
   id: string;
   tenantKey: string;
@@ -150,6 +146,16 @@ export async function upsertFirstPartyAnalyticsRollups(
   }
 
   const writeRollups = async (tx: any) => {
+    if (isPostgres()) {
+      // The historical backfill takes this lock before its raw-event
+      // snapshot. Holding it through ingest's rollup increment prevents the
+      // backfill's monotonic upsert from losing a concurrently committed
+      // event.
+      await tx.execute({
+        sql: FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
+        args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
+      });
+    }
     const dailyRows = [...dailyRollups.values()];
     await tx
       .insert(schema.analyticsEventDailyRollups)

package/corpus/templates/analytics/server/lib/first-party-analytics.ts

@@ -1,4 +1,5 @@
-import { getDbExec, isPostgres } from "@agent-native/core/db";
+import { getDbExec } from "@agent-native/core/db";
+import { runWithRequestContext } from "@agent-native/core/server";
 import { and, eq, isNull, or } from "drizzle-orm";
 
 import { FIRST_PARTY_ANALYTICS_QUERY_TIMEOUT_MS } from "../../shared/dashboard-report-timeouts.js";
@@ -8,6 +9,12 @@ import {
   ingestAnalyticsExceptionEvents,
   type DerivedExceptionFields,
 } from "./error-capture.js";
+import {
+  getFirstPartyAnalyticsBackend,
+  getFirstPartyAnalyticsTable,
+  insertFirstPartyAnalyticsRows,
+  queryFirstPartyAnalyticsInBigQuery,
+} from "./first-party-analytics-backend.js";
 import {
   firstPartyCacheKey,
   withFirstPartyCache,
@@ -17,11 +24,7 @@ import {
   queryOutcomeFromError,
   recordFirstPartyAnalyticsQueryPressure,
 } from "./first-party-analytics-health.js";
-import {
-  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY,
-  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
-  upsertFirstPartyAnalyticsRollups,
-} from "./first-party-analytics-rollups.js";
+import { upsertFirstPartyAnalyticsRollups } from "./first-party-analytics-rollups.js";
 
 export interface AnalyticsScope {
   userEmail: string;
@@ -508,18 +511,33 @@ export async function recordAnalyticsEvents(
     };
   });
 
-  if (rows.length) {
+  const backend = await getFirstPartyAnalyticsBackend({
+    userEmail: key.ownerEmail,
+    orgId: key.orgId ?? null,
+  });
+
+  if (rows.length && (backend.sink === "dual" || backend.sink === "bigquery")) {
+    try {
+      await runWithRequestContext(
+        {
+          userEmail: key.ownerEmail,
+          orgId: key.orgId ?? undefined,
+        },
+        () => insertFirstPartyAnalyticsRows(rows, backend.table),
+      );
+    } catch (error) {
+      if (backend.sink === "bigquery") throw error;
+      // Dual-write mode keeps Postgres as the recoverable source until the
+      // backfill has completed. A BigQuery outage must not lose live events.
+      console.error(
+        "[first-party-analytics] BigQuery dual-write failed; retaining Postgres event:",
+        error,
+      );
+    }
+  }
+
+  if (rows.length && backend.sink !== "bigquery") {
     await db.transaction(async (tx: any) => {
-      if (isPostgres()) {
-        // The historical backfill takes this lock before its raw-event
-        // snapshot. Holding it through ingest's rollup increment prevents the
-        // backfill's monotonic upsert from losing a concurrently committed
-        // event.
-        await tx.execute({
-          sql: FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
-          args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
-        });
-      }
       await tx.insert(schema.analyticsEvents).values(rows);
       await tx
         .update(schema.analyticsPublicKeys)
@@ -528,6 +546,12 @@ export async function recordAnalyticsEvents(
       await upsertFirstPartyAnalyticsRollups(rows, tx);
     });
   }
+  if (rows.length && backend.sink === "bigquery") {
+    await db
+      .update(schema.analyticsPublicKeys)
+      .set({ lastUsedAt: receivedAt })
+      .where(eq(schema.analyticsPublicKeys.id, key.id));
+  }
 
   // Fork captured exceptions into the dedicated error-capture tables. This is
   // best-effort: a malformed `$exception` payload must never reject the whole
@@ -984,6 +1008,24 @@ export async function queryFirstPartyAnalytics(
   options: AnalyticsQueryOptions = {},
 ): Promise<AnalyticsQueryResult> {
   validateFirstPartyAnalyticsSql(sql);
+  const backend = await getFirstPartyAnalyticsBackend(scope);
+  if (backend.sink === "bigquery") {
+    const usesSessionRecordings = /\bsession_recordings\b/i.test(sql);
+    const usesEventTables =
+      /\banalytics_events\b|\banalytics_event_daily_rollups\b|\banalytics_user_days\b/i.test(
+        sql,
+      );
+    if (usesSessionRecordings && usesEventTables) {
+      throw new Error(
+        "Cross-backend joins are not supported; query first-party event tables in BigQuery and session_recordings in the Analytics SQL store separately.",
+      );
+    }
+    if (!usesSessionRecordings) {
+      const table = await getFirstPartyAnalyticsTable(backend.table);
+      const scoped = scopedAnalyticsSql(sql, scope);
+      return queryFirstPartyAnalyticsInBigQuery(scoped.sql, scoped.args, table);
+    }
+  }
   const scoped = scopedAnalyticsSql(sql, scope);
   const wrappedSql = `SELECT * FROM (${scoped.sql}) AS first_party_analytics_query LIMIT ${MAX_QUERY_ROWS}`;
   const timeoutMs = Math.max(

package/corpus/templates/analytics/server/lib/real-data-actions.ts

@@ -65,7 +65,7 @@ export const CORPUS_REDUCTION_ACTIONS = new Set(["run-code"]);
 // hasDataQueryAttempt, but they should stop the guard from steering a
 // template-clone turn into "connect a missing source". Deliberately limited
 // to read/inspection actions: update-dashboard/mutate-dashboard/
-// compose-dashboard/install-dashboard-template/create-extension/
+// compose-dashboard/create-extension/
 // update-extension can all author brand-new SQL or extension content, so
 // calling one of those alone is not proof the turn actually inspected a
 // template rather than inventing it from scratch. If the tool run also
@@ -74,7 +74,6 @@ export const CORPUS_REDUCTION_ACTIONS = new Set(["run-code"]);
 export const DASHBOARD_CONSTRUCTION_ACTIONS = new Set([
   "get-sql-dashboard",
   "list-sql-dashboards",
-  "list-dashboard-templates",
   "list-extensions",
   "get-extension",
 ]);
@@ -88,7 +87,6 @@ export const DASHBOARD_MUTATION_ACTIONS = new Set([
   "mutate-dashboard",
   "update-dashboard",
   "compose-dashboard",
-  "install-dashboard-template",
   "create-extension",
   "update-extension",
 ]);

package/corpus/templates/analytics/server/plugins/agent-chat.ts

@@ -53,7 +53,6 @@ function hasSuccessfulDashboardSave(
     "update-dashboard",
     "mutate-dashboard",
     "compose-dashboard",
-    "install-dashboard-template",
   ]);
   return (toolResults ?? []).some((result) => {
     if (result.isError) return false;
@@ -737,9 +736,8 @@ export function realDataFinalGuard(
         'This is a dashboard construction/template-clone request. Resolve the named template\'s id (use `list-sql-dashboards` if you only have a title) and call `get-sql-dashboard` with `includeConfig: true` first. If its panels are `chartType: "extension"`, use `get-extension` then `create-extension` to clone/adapt it, then `update-dashboard` to save the new dashboard. Do not invent SQL panels for an extension-backed template. Ask one clarifying filter question if needed. Only run a data-source query before presenting numbers or authoring invented SQL.',
       fallbackMessage:
         "I need to inspect the template dashboard (and its extension, if it uses one) before creating the new one. Tell me the template dashboard name, or confirm the org/account filter, and I'll clone it without inventing metrics.",
-      // list-sql-dashboards/list-dashboard-templates are on the initial
-      // surface, but expand anyway so a corrective retry can always reach
-      // the lookup/inspection tools this message asks for.
+      // Expand the tool surface so a corrective retry can always reach the
+      // lookup/inspection tools this message asks for.
       expandToolSurface: true,
     };
   }

package/corpus/templates/analytics/server/plugins/analytics-rollup-backfill-jobs.ts

@@ -0,0 +1,61 @@
+import { runAnalyticsRollupBackfillOnce } from "../jobs/analytics-rollup-backfill";
+
+const DEFAULT_INTERVAL_MS = 15 * 60 * 1000;
+let skippingLogged = false;
+
+declare global {
+  var __AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__:
+    | boolean
+    | undefined;
+}
+
+function platformSchedulerOwnsBackfill(): boolean {
+  return (
+    process.env.NETLIFY === "true" ||
+    Boolean(process.env.NETLIFY_FUNCTION_NAME) ||
+    globalThis.__AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__ ===
+      true
+  );
+}
+
+function intervalMs(): number {
+  const raw = process.env.ANALYTICS_ROLLUP_BACKFILL_INTERVAL_MS?.trim();
+  if (!raw) return DEFAULT_INTERVAL_MS;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed >= 10_000
+    ? parsed
+    : DEFAULT_INTERVAL_MS;
+}
+
+export default function registerAnalyticsRollupBackfillJobs(): void {
+  const isProd = process.env.NODE_ENV === "production";
+  const flag =
+    process.env.ANALYTICS_ROLLUP_BACKFILL_JOBS ??
+    process.env.RUN_BACKGROUND_JOBS;
+  const enabled =
+    !platformSchedulerOwnsBackfill() &&
+    (flag === "1" || (isProd && flag !== "0"));
+
+  if (!enabled) {
+    if (!skippingLogged) {
+      console.log(
+        platformSchedulerOwnsBackfill()
+          ? "[analytics-rollups] Skipping in-process cron because the platform scheduler owns historical rollup backfills."
+          : "[analytics-rollups] Skipping historical rollup backfill (set ANALYTICS_ROLLUP_BACKFILL_JOBS=1 or RUN_BACKGROUND_JOBS=1 to enable in dev; on by default in production)",
+      );
+      skippingLogged = true;
+    }
+    return;
+  }
+
+  const ms = intervalMs();
+  setInterval(() => {
+    runAnalyticsRollupBackfillOnce().catch((err) =>
+      console.error("[analytics-rollups] historical backfill failed:", err),
+    );
+  }, ms);
+
+  console.log(
+    `[analytics-rollups] Historical rollup backfill scheduled every ${ms / 1000}s.`,
+  );
+}