@agent-native/core 0.141.4 → 0.141.6

package/corpus/templates/analytics/server/lib/dashboard-catalog.ts

@@ -1,6 +1,10 @@
 import type { SqlDashboardConfig } from "../../app/pages/adhoc/sql-dashboard/types";
 import { loadDashboardSeed } from "./dashboard-seeds";
-import { listDashboards, type DashboardRecord } from "./dashboards-store";
+import {
+  listDashboardSummaries,
+  type DashboardRecord,
+  type DashboardSummaryRecord,
+} from "./dashboards-store";
 import {
   FIRST_PARTY_DASHBOARD_ID,
   repairFirstPartyObservedRetentionPanels,
@@ -1021,33 +1025,13 @@ export function cloneDashboardConfig(
   return JSON.parse(JSON.stringify(entry.buildConfig())) as SqlDashboardConfig;
 }
 
-function templateIdFromConfig(config: Record<string, unknown>): string | null {
-  const catalog = config.catalog;
-  if (!catalog || typeof catalog !== "object" || Array.isArray(catalog)) {
-    return null;
-  }
-  const templateId = (catalog as Record<string, unknown>).templateId;
-  return typeof templateId === "string" && templateId ? templateId : null;
-}
-
-function demoIdFromConfig(config: Record<string, unknown>): string | null {
-  const demo = config.demo;
-  if (!demo || typeof demo !== "object" || Array.isArray(demo)) {
-    return null;
-  }
-  const demoId = (demo as Record<string, unknown>).id;
-  return typeof demoId === "string" && demoId ? demoId : null;
-}
-
 function installedDashboardForTemplate(
-  row: DashboardRecord,
+  row: DashboardSummaryRecord,
   entry: DashboardCatalogEntry,
 ): boolean {
-  const templateId = templateIdFromConfig(row.config);
-  const demoId = demoIdFromConfig(row.config);
   return (
-    templateId === entry.id ||
-    demoId === entry.id ||
+    row.catalogTemplateId === entry.id ||
+    row.demoId === entry.id ||
     row.id === entry.defaultDashboardId
   );
 }
@@ -1055,10 +1039,11 @@ function installedDashboardForTemplate(
 export async function listDashboardCatalog(
   ctx: AccessCtx,
 ): Promise<DashboardCatalogTemplate[]> {
-  const dashboards = await listDashboards(ctx, {
+  const dashboards = await listDashboardSummaries(ctx, {
     kind: "sql",
     archived: "all",
     hidden: "all",
+    includeCatalogMetadata: true,
   });
 
   return dashboardCatalogEntries
@@ -1068,10 +1053,7 @@ export async function listDashboardCatalog(
         .filter((row) => installedDashboardForTemplate(row, entry))
         .map((row) => ({
           id: row.id,
-          name:
-            typeof row.config.name === "string" && row.config.name.trim()
-              ? row.config.name
-              : row.title,
+          name: row.configName?.trim() || row.name,
           visibility: row.visibility,
           updatedAt: row.updatedAt,
           archivedAt: row.archivedAt,

package/corpus/templates/analytics/server/lib/dashboards-store.ts

@@ -62,6 +62,9 @@ export interface DashboardSummaryRecord {
   id: string;
   kind: DashboardKind;
   name: string;
+  configName: string | null;
+  catalogTemplateId: string | null;
+  demoId: string | null;
   parentId: string | null;
   ownerEmail: string;
   orgId: string | null;
@@ -287,6 +290,24 @@ function configFromSettings(data: Record<string, unknown>): {
   return { title, config: data };
 }
 
+function stringProperty(value: unknown, key: string): string | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const property = (value as Record<string, unknown>)[key];
+  return typeof property === "string" ? property : null;
+}
+
+function catalogMetadataFromConfig(config: Record<string, unknown>): {
+  configName: string | null;
+  catalogTemplateId: string | null;
+  demoId: string | null;
+} {
+  return {
+    configName: typeof config.name === "string" ? config.name : null,
+    catalogTemplateId: stringProperty(config.catalog, "templateId"),
+    demoId: stringProperty(config.demo, "id"),
+  };
+}
+
 async function migrateDashboardFromSettings(
   id: string,
   kind: DashboardKind,
@@ -510,11 +531,13 @@ export async function listDashboardSummaries(
     kind?: DashboardKind;
     archived?: DashboardArchiveFilter;
     hidden?: DashboardHiddenFilter;
+    includeCatalogMetadata?: boolean;
   },
 ): Promise<DashboardSummaryRecord[]> {
   const db = getDb() as any;
   const archived = filter?.archived ?? "active";
   const hidden = filter?.hidden ?? "visible";
+  const includeCatalogMetadata = filter?.includeCatalogMetadata === true;
   const conditions: any[] = [
     accessFilter(schema.dashboards, schema.dashboardShares, {
       userEmail: ctx.email,
@@ -535,11 +558,31 @@ export async function listDashboardSummaries(
     : sql<
         string | null
       >`json_extract(${schema.dashboards.config}, '$.parentId')`;
+  const configName = isPostgres()
+    ? sql<string | null>`(${schema.dashboards.config}::jsonb ->> 'name')`
+    : sql<string | null>`json_extract(${schema.dashboards.config}, '$.name')`;
+  const catalogTemplateId = isPostgres()
+    ? sql<
+        string | null
+      >`(${schema.dashboards.config}::jsonb -> 'catalog' ->> 'templateId')`
+    : sql<
+        string | null
+      >`json_extract(${schema.dashboards.config}, '$.catalog.templateId')`;
+  const demoId = isPostgres()
+    ? sql<
+        string | null
+      >`(${schema.dashboards.config}::jsonb -> 'demo' ->> 'id')`
+    : sql<
+        string | null
+      >`json_extract(${schema.dashboards.config}, '$.demo.id')`;
   const rows = await db
     .select({
       id: schema.dashboards.id,
       kind: schema.dashboards.kind,
       name: schema.dashboards.title,
+      ...(includeCatalogMetadata
+        ? { configName, catalogTemplateId, demoId }
+        : {}),
       parentId,
       ownerEmail: schema.dashboards.ownerEmail,
       orgId: schema.dashboards.orgId,
@@ -554,6 +597,10 @@ export async function listDashboardSummaries(
     .where(where);
   const out: DashboardSummaryRecord[] = rows.map((row: any) => ({
     ...row,
+    configName: typeof row.configName === "string" ? row.configName : null,
+    catalogTemplateId:
+      typeof row.catalogTemplateId === "string" ? row.catalogTemplateId : null,
+    demoId: typeof row.demoId === "string" ? row.demoId : null,
     parentId: typeof row.parentId === "string" ? row.parentId : null,
     orgId: row.orgId ?? null,
     archivedAt: row.archivedAt ?? null,
@@ -590,12 +637,20 @@ export async function listDashboardSummaries(
       seen.add(id);
       const config = value as Record<string, unknown>;
       const { title } = configFromSettings(config);
+      const catalogMetadata = includeCatalogMetadata
+        ? catalogMetadataFromConfig(config)
+        : {
+            configName: null,
+            catalogTemplateId: null,
+            demoId: null,
+          };
       const createdAt =
         typeof config.createdAt === "string" ? config.createdAt : nowIso();
       out.push({
         id,
         kind,
         name: title,
+        ...catalogMetadata,
         parentId: typeof config.parentId === "string" ? config.parentId : null,
         ownerEmail: ctx.email,
         orgId,

package/corpus/templates/analytics/server/lib/first-party-analytics-health.ts

@@ -13,11 +13,33 @@ export const FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS = {
   recommendMaxQueryMs: 30_000,
 } as const;
 
-const BIGQUERY_SETUP_LINK = buildDeepLink({
-  app: "analytics",
-  view: "data-sources",
-  to: "/data-sources?source=bigquery",
-});
+const EXTERNAL_BACKEND_DEFINITIONS = [
+  {
+    id: "bigquery",
+    label: "BigQuery",
+    role: "warehouse",
+    requiredKeys: [
+      "GOOGLE_APPLICATION_CREDENTIALS_JSON",
+      "BIGQUERY_PROJECT_ID",
+    ],
+    setupLink: buildDeepLink({
+      app: "analytics",
+      view: "data-sources",
+      to: "/data-sources?source=bigquery",
+    }),
+  },
+  {
+    id: "amplitude",
+    label: "Amplitude",
+    role: "product-analytics",
+    requiredKeys: ["AMPLITUDE_API_KEY", "AMPLITUDE_SECRET_KEY"],
+    setupLink: buildDeepLink({
+      app: "analytics",
+      view: "data-sources",
+      to: "/data-sources?source=amplitude",
+    }),
+  },
+] as const;
 
 export type FirstPartyAnalyticsQueryClass =
   | "raw-events"
@@ -39,9 +61,29 @@ export type FirstPartyAnalyticsRecommendation =
   | "connect_bigquery"
   | "use_bigquery";
 
+export type FirstPartyAnalyticsExternalBackendRecommendation =
+  | "none"
+  | "connect"
+  | "use"
+  | "unknown";
+
+export type FirstPartyAnalyticsBackendId =
+  (typeof EXTERNAL_BACKEND_DEFINITIONS)[number]["id"];
+
+export interface FirstPartyAnalyticsBackendStatus {
+  id: FirstPartyAnalyticsBackendId;
+  label: string;
+  role: (typeof EXTERNAL_BACKEND_DEFINITIONS)[number]["role"];
+  configured: boolean | null;
+  missingRequiredKeys: string[];
+  setupLink: string;
+}
+
 export interface FirstPartyAnalyticsHealth {
   status: FirstPartyAnalyticsHealthStatus;
   recommendation: FirstPartyAnalyticsRecommendation;
+  externalBackendRecommendation: FirstPartyAnalyticsExternalBackendRecommendation;
+  externalBackends: FirstPartyAnalyticsBackendStatus[];
   reasons: Array<"event_volume" | "slow_queries" | "query_timeout">;
   observedAt: string;
   metrics: {
@@ -56,11 +98,8 @@ export interface FirstPartyAnalyticsHealth {
     maxQueryDurationMs24h: number;
   };
   thresholds: typeof FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS;
-  bigQuery: {
-    configured: boolean | null;
-    missingRequiredKeys: string[];
-    setupLink: string;
-  };
+  /** Kept for clients that still read the original BigQuery-only field. */
+  bigQuery: FirstPartyAnalyticsBackendStatus;
 }
 
 interface QueryPressureEvent {
@@ -199,18 +238,13 @@ function dateSpanDays(
   return Math.floor((last - first) / 86_400_000) + 1;
 }
 
-async function bigQueryStatus(scope: AnalyticsScope): Promise<{
-  configured: boolean | null;
-  missingRequiredKeys: string[];
-  setupLink: string;
-}> {
-  const requiredKeys = [
-    "GOOGLE_APPLICATION_CREDENTIALS_JSON",
-    "BIGQUERY_PROJECT_ID",
-  ];
+async function externalBackendStatus(
+  scope: AnalyticsScope,
+  definition: (typeof EXTERNAL_BACKEND_DEFINITIONS)[number],
+): Promise<FirstPartyAnalyticsBackendStatus> {
   try {
     const configured = await Promise.all(
-      requiredKeys.map((key) =>
+      definition.requiredKeys.map((key) =>
         hasCredential(key, {
           userEmail: scope.userEmail,
           orgId: scope.orgId,
@@ -218,22 +252,41 @@ async function bigQueryStatus(scope: AnalyticsScope): Promise<{
       ),
     );
     return {
+      id: definition.id,
+      label: definition.label,
+      role: definition.role,
       configured: configured.every(Boolean),
-      missingRequiredKeys: requiredKeys.filter(
+      missingRequiredKeys: definition.requiredKeys.filter(
         (_, index) => !configured[index],
       ),
-      setupLink: BIGQUERY_SETUP_LINK,
+      setupLink: definition.setupLink,
     };
   } catch (error) {
-    console.warn("[first-party-analytics] BigQuery status unavailable:", error);
+    console.warn(
+      `[first-party-analytics] ${definition.label} status unavailable:`,
+      error,
+    );
     return {
+      id: definition.id,
+      label: definition.label,
+      role: definition.role,
       configured: null,
       missingRequiredKeys: [],
-      setupLink: BIGQUERY_SETUP_LINK,
+      setupLink: definition.setupLink,
     };
   }
 }
 
+async function externalBackendStatuses(
+  scope: AnalyticsScope,
+): Promise<FirstPartyAnalyticsBackendStatus[]> {
+  return Promise.all(
+    EXTERNAL_BACKEND_DEFINITIONS.map((definition) =>
+      externalBackendStatus(scope, definition),
+    ),
+  );
+}
+
 function recommendationFor(
   status: FirstPartyAnalyticsHealthStatus,
   configured: boolean | null,
@@ -242,6 +295,18 @@ function recommendationFor(
   return configured === true ? "use_bigquery" : "connect_bigquery";
 }
 
+function externalBackendRecommendationFor(
+  status: FirstPartyAnalyticsHealthStatus,
+  backends: FirstPartyAnalyticsBackendStatus[],
+): FirstPartyAnalyticsExternalBackendRecommendation {
+  if (status !== "recommend_bigquery") return "none";
+  if (backends.some((backend) => backend.configured === true)) return "use";
+  if (backends.every((backend) => backend.configured === false)) {
+    return "connect";
+  }
+  return "unknown";
+}
+
 export async function getFirstPartyAnalyticsHealth(
   scope: AnalyticsScope,
 ): Promise<FirstPartyAnalyticsHealth> {
@@ -256,7 +321,7 @@ export async function getFirstPartyAnalyticsHealth(
   const pressure = schema.analyticsQueryPressureDaily;
   const db = getDb() as any;
 
-  const [rollupRows, pressureRows, bigQuery] = await Promise.all([
+  const [rollupRows, pressureRows, externalBackends] = await Promise.all([
     db
       .select({
         eventCount: sql<number>`coalesce(sum(${rollups.eventCount}), 0)`,
@@ -284,9 +349,14 @@ export async function getFirstPartyAnalyticsHealth(
           lte(pressure.lastSeenAt, nowTimestamp),
         ),
       ),
-    bigQueryStatus(scope),
+    externalBackendStatuses(scope),
   ]);
 
+  const bigQuery = externalBackends.find(
+    (backend) => backend.id === "bigquery",
+  );
+  if (!bigQuery) throw new Error("BigQuery backend status is missing");
+
   const rollup = (rollupRows[0] ?? {}) as Record<string, unknown>;
   const eventCount = finiteNumber(rollup.eventCount);
   const dailyRollupRows = finiteNumber(rollup.dailyRollupRows);
@@ -348,6 +418,11 @@ export async function getFirstPartyAnalyticsHealth(
   return {
     status,
     recommendation: recommendationFor(status, bigQuery.configured),
+    externalBackendRecommendation: externalBackendRecommendationFor(
+      status,
+      externalBackends,
+    ),
+    externalBackends,
     reasons,
     observedAt: nowIso(),
     metrics: {
@@ -367,9 +442,24 @@ export async function getFirstPartyAnalyticsHealth(
 }
 
 export function unavailableFirstPartyAnalyticsHealth(): FirstPartyAnalyticsHealth {
+  const externalBackends = EXTERNAL_BACKEND_DEFINITIONS.map((definition) => ({
+    id: definition.id,
+    label: definition.label,
+    role: definition.role,
+    configured: null,
+    missingRequiredKeys: [],
+    setupLink: definition.setupLink,
+  }));
+  const bigQuery = externalBackends.find(
+    (backend) => backend.id === "bigquery",
+  );
+  if (!bigQuery) throw new Error("BigQuery backend status is missing");
+
   return {
     status: "unavailable",
     recommendation: "none",
+    externalBackendRecommendation: "unknown",
+    externalBackends,
     reasons: [],
     observedAt: nowIso(),
     metrics: {
@@ -384,10 +474,6 @@ export function unavailableFirstPartyAnalyticsHealth(): FirstPartyAnalyticsHealt
       maxQueryDurationMs24h: 0,
     },
     thresholds: FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS,
-    bigQuery: {
-      configured: null,
-      missingRequiredKeys: [],
-      setupLink: BIGQUERY_SETUP_LINK,
-    },
+    bigQuery,
   };
 }

package/corpus/templates/analytics/server/lib/first-party-analytics-rollups.ts

@@ -1,6 +1,11 @@
+import { isPostgres } from "@agent-native/core/db";
 import { sql } from "@agent-native/core/db/schema";
 
 import { getDb, schema } from "../db/index.js";
+import {
+  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY,
+  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
+} from "./analytics-rollup-lock.js";
 
 /**
  * The normalized subset emitted by first-party analytics ingest. Keeping this
@@ -23,15 +28,6 @@ export interface FirstPartyAnalyticsRollupResult {
   userDayCount: number;
 }
 
-/**
- * Shared with the historical backfill so live ingest and migration work hold
- * the same transaction-scoped Postgres lock while they update rollups.
- */
-export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY =
-  "agent-native:analytics-rollup-backfill";
-export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL =
-  "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))";
-
 interface DailyRollupRow {
   id: string;
   tenantKey: string;
@@ -150,6 +146,16 @@ export async function upsertFirstPartyAnalyticsRollups(
   }
 
   const writeRollups = async (tx: any) => {
+    if (isPostgres()) {
+      // The historical backfill takes this lock before its raw-event
+      // snapshot. Holding it through ingest's rollup increment prevents the
+      // backfill's monotonic upsert from losing a concurrently committed
+      // event.
+      await tx.execute({
+        sql: FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
+        args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
+      });
+    }
     const dailyRows = [...dailyRollups.values()];
     await tx
       .insert(schema.analyticsEventDailyRollups)

package/corpus/templates/analytics/server/lib/first-party-analytics.ts

@@ -1,4 +1,4 @@
-import { getDbExec, isPostgres } from "@agent-native/core/db";
+import { getDbExec } from "@agent-native/core/db";
 import { and, eq, isNull, or } from "drizzle-orm";
 
 import { FIRST_PARTY_ANALYTICS_QUERY_TIMEOUT_MS } from "../../shared/dashboard-report-timeouts.js";
@@ -17,11 +17,7 @@ import {
   queryOutcomeFromError,
   recordFirstPartyAnalyticsQueryPressure,
 } from "./first-party-analytics-health.js";
-import {
-  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY,
-  FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
-  upsertFirstPartyAnalyticsRollups,
-} from "./first-party-analytics-rollups.js";
+import { upsertFirstPartyAnalyticsRollups } from "./first-party-analytics-rollups.js";
 
 export interface AnalyticsScope {
   userEmail: string;
@@ -510,16 +506,9 @@ export async function recordAnalyticsEvents(
 
   if (rows.length) {
     await db.transaction(async (tx: any) => {
-      if (isPostgres()) {
-        // The historical backfill takes this lock before its raw-event
-        // snapshot. Holding it through ingest's rollup increment prevents the
-        // backfill's monotonic upsert from losing a concurrently committed
-        // event.
-        await tx.execute({
-          sql: FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL,
-          args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
-        });
-      }
+      // Raw events and incremental rollups share this transaction. The
+      // historical boot backfill is disabled, so no global cross-request lock
+      // is needed to coordinate a second writer.
       await tx.insert(schema.analyticsEvents).values(rows);
       await tx
         .update(schema.analyticsPublicKeys)

package/corpus/templates/analytics/server/plugins/analytics-rollup-backfill-jobs.ts

@@ -0,0 +1,61 @@
+import { runAnalyticsRollupBackfillOnce } from "../jobs/analytics-rollup-backfill";
+
+const DEFAULT_INTERVAL_MS = 15 * 60 * 1000;
+let skippingLogged = false;
+
+declare global {
+  var __AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__:
+    | boolean
+    | undefined;
+}
+
+function platformSchedulerOwnsBackfill(): boolean {
+  return (
+    process.env.NETLIFY === "true" ||
+    Boolean(process.env.NETLIFY_FUNCTION_NAME) ||
+    globalThis.__AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__ ===
+      true
+  );
+}
+
+function intervalMs(): number {
+  const raw = process.env.ANALYTICS_ROLLUP_BACKFILL_INTERVAL_MS?.trim();
+  if (!raw) return DEFAULT_INTERVAL_MS;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed >= 10_000
+    ? parsed
+    : DEFAULT_INTERVAL_MS;
+}
+
+export default function registerAnalyticsRollupBackfillJobs(): void {
+  const isProd = process.env.NODE_ENV === "production";
+  const flag =
+    process.env.ANALYTICS_ROLLUP_BACKFILL_JOBS ??
+    process.env.RUN_BACKGROUND_JOBS;
+  const enabled =
+    !platformSchedulerOwnsBackfill() &&
+    (flag === "1" || (isProd && flag !== "0"));
+
+  if (!enabled) {
+    if (!skippingLogged) {
+      console.log(
+        platformSchedulerOwnsBackfill()
+          ? "[analytics-rollups] Skipping in-process cron because the platform scheduler owns historical rollup backfills."
+          : "[analytics-rollups] Skipping historical rollup backfill (set ANALYTICS_ROLLUP_BACKFILL_JOBS=1 or RUN_BACKGROUND_JOBS=1 to enable in dev; on by default in production)",
+      );
+      skippingLogged = true;
+    }
+    return;
+  }
+
+  const ms = intervalMs();
+  setInterval(() => {
+    runAnalyticsRollupBackfillOnce().catch((err) =>
+      console.error("[analytics-rollups] historical backfill failed:", err),
+    );
+  }, ms);
+
+  console.log(
+    `[analytics-rollups] Historical rollup backfill scheduled every ${ms / 1000}s.`,
+  );
+}