@agent-native/core 0.137.7 → 0.138.0

package/corpus/templates/analytics/server/lib/first-party-analytics-health.ts

@@ -0,0 +1,397 @@
+import { sql } from "@agent-native/core/db/schema";
+import { buildDeepLink } from "@agent-native/core/server";
+import { and, gte, inArray, lte } from "drizzle-orm";
+
+import { getDb, schema } from "../db/index.js";
+import { hasCredential } from "./credentials.js";
+import type { AnalyticsScope } from "./first-party-analytics.js";
+
+export const FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS = {
+  slowQueryMs: 5_000,
+  recommendEventCount: 1_000_000,
+  recommendSlowQueries24h: 3,
+  recommendMaxQueryMs: 30_000,
+} as const;
+
+const BIGQUERY_SETUP_LINK = buildDeepLink({
+  app: "analytics",
+  view: "data-sources",
+  to: "/data-sources?source=bigquery",
+});
+
+export type FirstPartyAnalyticsQueryClass =
+  | "raw-events"
+  | "rollups"
+  | "session-replay"
+  | "mixed"
+  | "other";
+
+export type FirstPartyAnalyticsQueryOutcome = "success" | "timeout" | "error";
+
+export type FirstPartyAnalyticsHealthStatus =
+  | "healthy"
+  | "monitor"
+  | "recommend_bigquery"
+  | "unavailable";
+
+export type FirstPartyAnalyticsRecommendation =
+  | "none"
+  | "connect_bigquery"
+  | "use_bigquery";
+
+export interface FirstPartyAnalyticsHealth {
+  status: FirstPartyAnalyticsHealthStatus;
+  recommendation: FirstPartyAnalyticsRecommendation;
+  reasons: Array<"event_volume" | "slow_queries" | "query_timeout">;
+  observedAt: string;
+  metrics: {
+    eventCount: number;
+    dailyRollupRows: number;
+    firstEventDate: string | null;
+    lastEventDate: string | null;
+    spanDays: number;
+    slowQueryCount24h: number;
+    timeoutCount24h: number;
+    errorCount24h: number;
+    maxQueryDurationMs24h: number;
+  };
+  thresholds: typeof FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS;
+  bigQuery: {
+    configured: boolean | null;
+    missingRequiredKeys: string[];
+    setupLink: string;
+  };
+}
+
+interface QueryPressureEvent {
+  durationMs: number;
+  outcome: FirstPartyAnalyticsQueryOutcome;
+  queryClass: FirstPartyAnalyticsQueryClass;
+}
+
+function nowIso(): string {
+  return new Date().toISOString();
+}
+
+function todayIsoDate(): string {
+  return nowIso().slice(0, 10);
+}
+
+function tenantKey(scope: AnalyticsScope): string {
+  return scope.orgId ? `org:${scope.orgId}` : `user:${scope.userEmail}`;
+}
+
+function stableId(parts: readonly string[]): string {
+  return `aqpd_${parts.map((part) => encodeURIComponent(part)).join("|")}`;
+}
+
+function finiteNumber(value: unknown): number {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "bigint") return Number(value);
+  if (typeof value === "string" && value.trim()) {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return 0;
+}
+
+function normalizedDurationMs(value: number): number {
+  return Number.isFinite(value) ? Math.max(0, Math.round(value)) : 0;
+}
+
+function isTimeoutError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /timeout|timed out|statement deadline|cancelled by deadline/i.test(
+    message,
+  );
+}
+
+export function queryOutcomeFromError(
+  error: unknown,
+): FirstPartyAnalyticsQueryOutcome {
+  return isTimeoutError(error) ? "timeout" : "error";
+}
+
+export function classifyFirstPartyAnalyticsQuery(
+  query: string,
+): FirstPartyAnalyticsQueryClass {
+  const normalized = query.toLowerCase();
+  const usesRawEvents = /\banalytics_events\b/.test(normalized);
+  const usesRollups =
+    /\banalytics_event_daily_rollups\b|\banalytics_user_days\b/.test(
+      normalized,
+    );
+  const usesSessionReplay = /\bsession_recordings\b/.test(normalized);
+  const sourceCount =
+    Number(usesRawEvents) + Number(usesRollups) + Number(usesSessionReplay);
+
+  if (sourceCount > 1) return "mixed";
+  if (usesRawEvents) return "raw-events";
+  if (usesRollups) return "rollups";
+  if (usesSessionReplay) return "session-replay";
+  return "other";
+}
+
+function shouldRecordPressure(event: QueryPressureEvent): boolean {
+  return (
+    event.outcome !== "success" ||
+    normalizedDurationMs(event.durationMs) >=
+      FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.slowQueryMs
+  );
+}
+
+/**
+ * Persist only slow/failing query aggregates. This intentionally sits outside
+ * the raw event stream so diagnosing database pressure cannot recursively add
+ * more analytics events to the same database.
+ */
+export async function recordFirstPartyAnalyticsQueryPressure(
+  scope: AnalyticsScope,
+  event: QueryPressureEvent,
+): Promise<void> {
+  if (!shouldRecordPressure(event)) return;
+
+  const durationMs = normalizedDurationMs(event.durationMs);
+  const eventDate = todayIsoDate();
+  const key = tenantKey(scope);
+  const table = schema.analyticsQueryPressureDaily;
+  const db = getDb() as any;
+  const isSlow =
+    event.outcome !== "success" ||
+    durationMs >= FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.slowQueryMs;
+
+  await db
+    .insert(table)
+    .values({
+      id: stableId([key, eventDate, event.queryClass]),
+      tenantKey: key,
+      ownerEmail: scope.userEmail,
+      orgId: scope.orgId,
+      eventDate,
+      queryClass: event.queryClass,
+      slowQueryCount: isSlow ? 1 : 0,
+      timeoutCount: event.outcome === "timeout" ? 1 : 0,
+      errorCount: event.outcome === "error" ? 1 : 0,
+      totalDurationMs: durationMs,
+      maxDurationMs: durationMs,
+      lastSeenAt: nowIso(),
+    })
+    .onConflictDoUpdate({
+      target: [table.tenantKey, table.eventDate, table.queryClass],
+      set: {
+        slowQueryCount: sql`${table.slowQueryCount} + ${isSlow ? 1 : 0}`,
+        timeoutCount: sql`${table.timeoutCount} + ${event.outcome === "timeout" ? 1 : 0}`,
+        errorCount: sql`${table.errorCount} + ${event.outcome === "error" ? 1 : 0}`,
+        totalDurationMs: sql`${table.totalDurationMs} + ${durationMs}`,
+        maxDurationMs: sql`CASE WHEN ${table.maxDurationMs} > ${durationMs} THEN ${table.maxDurationMs} ELSE ${durationMs} END`,
+        lastSeenAt: nowIso(),
+      },
+    });
+}
+
+function dateSpanDays(
+  firstEventDate: string | null,
+  lastEventDate: string | null,
+): number {
+  if (!firstEventDate || !lastEventDate) return 0;
+  const first = Date.parse(`${firstEventDate}T00:00:00.000Z`);
+  const last = Date.parse(`${lastEventDate}T00:00:00.000Z`);
+  if (!Number.isFinite(first) || !Number.isFinite(last) || last < first) {
+    return 0;
+  }
+  return Math.floor((last - first) / 86_400_000) + 1;
+}
+
+async function bigQueryStatus(scope: AnalyticsScope): Promise<{
+  configured: boolean | null;
+  missingRequiredKeys: string[];
+  setupLink: string;
+}> {
+  const requiredKeys = [
+    "GOOGLE_APPLICATION_CREDENTIALS_JSON",
+    "BIGQUERY_PROJECT_ID",
+  ];
+  try {
+    const configured = await Promise.all(
+      requiredKeys.map((key) =>
+        hasCredential(key, {
+          userEmail: scope.userEmail,
+          orgId: scope.orgId,
+        }),
+      ),
+    );
+    return {
+      configured: configured.every(Boolean),
+      missingRequiredKeys: requiredKeys.filter(
+        (_, index) => !configured[index],
+      ),
+      setupLink: BIGQUERY_SETUP_LINK,
+    };
+  } catch (error) {
+    console.warn("[first-party-analytics] BigQuery status unavailable:", error);
+    return {
+      configured: null,
+      missingRequiredKeys: [],
+      setupLink: BIGQUERY_SETUP_LINK,
+    };
+  }
+}
+
+function recommendationFor(
+  status: FirstPartyAnalyticsHealthStatus,
+  configured: boolean | null,
+): FirstPartyAnalyticsRecommendation {
+  if (status !== "recommend_bigquery") return "none";
+  return configured === true ? "use_bigquery" : "connect_bigquery";
+}
+
+export async function getFirstPartyAnalyticsHealth(
+  scope: AnalyticsScope,
+): Promise<FirstPartyAnalyticsHealth> {
+  const now = new Date();
+  const today = now.toISOString().slice(0, 10);
+  const since24h = new Date(now.getTime() - 86_400_000)
+    .toISOString()
+    .slice(0, 10);
+  const keys = scope.orgId
+    ? [`org:${scope.orgId}`, `user:${scope.userEmail}`]
+    : [`user:${scope.userEmail}`];
+  const rollups = schema.analyticsEventDailyRollups;
+  const pressure = schema.analyticsQueryPressureDaily;
+  const db = getDb() as any;
+
+  const [rollupRows, pressureRows, bigQuery] = await Promise.all([
+    db
+      .select({
+        eventCount: sql<number>`coalesce(sum(${rollups.eventCount}), 0)`,
+        dailyRollupRows: sql<number>`count(*)`,
+        firstEventDate: sql<string | null>`min(${rollups.eventDate})`,
+        lastEventDate: sql<string | null>`max(${rollups.eventDate})`,
+      })
+      .from(rollups)
+      .where(
+        and(inArray(rollups.tenantKey, keys), lte(rollups.eventDate, today)),
+      ),
+    db
+      .select({
+        queryClass: pressure.queryClass,
+        slowQueryCount: pressure.slowQueryCount,
+        timeoutCount: pressure.timeoutCount,
+        errorCount: pressure.errorCount,
+        maxDurationMs: pressure.maxDurationMs,
+      })
+      .from(pressure)
+      .where(
+        and(
+          inArray(pressure.tenantKey, keys),
+          gte(pressure.eventDate, since24h),
+          lte(pressure.eventDate, today),
+        ),
+      ),
+    bigQueryStatus(scope),
+  ]);
+
+  const rollup = (rollupRows[0] ?? {}) as Record<string, unknown>;
+  const eventCount = finiteNumber(rollup.eventCount);
+  const dailyRollupRows = finiteNumber(rollup.dailyRollupRows);
+  const firstEventDate =
+    typeof rollup.firstEventDate === "string" ? rollup.firstEventDate : null;
+  const lastEventDate =
+    typeof rollup.lastEventDate === "string" ? rollup.lastEventDate : null;
+  const slowQueryCount24h = pressureRows.reduce(
+    (sum: number, row: Record<string, unknown>) =>
+      sum + finiteNumber(row.slowQueryCount),
+    0,
+  );
+  const timeoutCount24h = pressureRows.reduce(
+    (sum: number, row: Record<string, unknown>) =>
+      sum + finiteNumber(row.timeoutCount),
+    0,
+  );
+  const errorCount24h = pressureRows.reduce(
+    (sum: number, row: Record<string, unknown>) =>
+      sum + finiteNumber(row.errorCount),
+    0,
+  );
+  const maxQueryDurationMs24h = pressureRows.reduce(
+    (max: number, row: Record<string, unknown>) =>
+      Math.max(max, finiteNumber(row.maxDurationMs)),
+    0,
+  );
+  const reasons: FirstPartyAnalyticsHealth["reasons"] = [];
+  if (
+    eventCount >= FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.recommendEventCount
+  ) {
+    reasons.push("event_volume");
+  }
+  if (
+    slowQueryCount24h >=
+    FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.recommendSlowQueries24h
+  ) {
+    reasons.push("slow_queries");
+  }
+  if (
+    timeoutCount24h > 0 ||
+    maxQueryDurationMs24h >=
+      FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.recommendMaxQueryMs
+  ) {
+    reasons.push("query_timeout");
+  }
+
+  const hasMonitorSignal =
+    eventCount >=
+      FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS.recommendEventCount / 4 ||
+    slowQueryCount24h > 0;
+  const status: FirstPartyAnalyticsHealthStatus =
+    reasons.length > 0
+      ? "recommend_bigquery"
+      : hasMonitorSignal
+        ? "monitor"
+        : "healthy";
+
+  return {
+    status,
+    recommendation: recommendationFor(status, bigQuery.configured),
+    reasons,
+    observedAt: nowIso(),
+    metrics: {
+      eventCount,
+      dailyRollupRows,
+      firstEventDate,
+      lastEventDate,
+      spanDays: dateSpanDays(firstEventDate, lastEventDate),
+      slowQueryCount24h,
+      timeoutCount24h,
+      errorCount24h,
+      maxQueryDurationMs24h,
+    },
+    thresholds: FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS,
+    bigQuery,
+  };
+}
+
+export function unavailableFirstPartyAnalyticsHealth(): FirstPartyAnalyticsHealth {
+  return {
+    status: "unavailable",
+    recommendation: "none",
+    reasons: [],
+    observedAt: nowIso(),
+    metrics: {
+      eventCount: 0,
+      dailyRollupRows: 0,
+      firstEventDate: null,
+      lastEventDate: null,
+      spanDays: 0,
+      slowQueryCount24h: 0,
+      timeoutCount24h: 0,
+      errorCount24h: 0,
+      maxQueryDurationMs24h: 0,
+    },
+    thresholds: FIRST_PARTY_ANALYTICS_PRESSURE_THRESHOLDS,
+    bigQuery: {
+      configured: null,
+      missingRequiredKeys: [],
+      setupLink: BIGQUERY_SETUP_LINK,
+    },
+  };
+}

package/corpus/templates/analytics/server/lib/first-party-analytics-rollups.ts

@@ -0,0 +1,182 @@
+import { sql } from "@agent-native/core/db/schema";
+
+import { getDb, schema } from "../db/index.js";
+
+/**
+ * The normalized subset emitted by first-party analytics ingest. Keeping this
+ * contract smaller than the raw event row makes it clear that rollups never
+ * need to parse properties or context JSON.
+ */
+export interface NormalizedFirstPartyAnalyticsEventRow {
+  eventName: string;
+  eventDate: string;
+  ownerEmail: string;
+  orgId?: string | null;
+  userKey?: string | null;
+  app?: string | null;
+  template?: string | null;
+}
+
+export interface FirstPartyAnalyticsRollupResult {
+  eventCount: number;
+  dailyRollupCount: number;
+  userDayCount: number;
+}
+
+interface DailyRollupRow {
+  id: string;
+  tenantKey: string;
+  ownerEmail: string;
+  orgId: string | null;
+  eventDate: string;
+  eventName: string;
+  app: string;
+  template: string;
+  eventCount: number;
+}
+
+interface UserDayRow {
+  id: string;
+  tenantKey: string;
+  ownerEmail: string;
+  orgId: string | null;
+  eventDate: string;
+  userKey: string;
+}
+
+function requiredString(value: unknown, field: string): string {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new Error(`Analytics rollup row requires a non-empty ${field}`);
+  }
+  return value.trim();
+}
+
+function nullableString(value: unknown, field: string): string | null {
+  if (value == null) return null;
+  if (typeof value !== "string") {
+    throw new Error(`Analytics rollup row ${field} must be a string or null`);
+  }
+  const trimmed = value.trim();
+  return trimmed || null;
+}
+
+function tenantKey(ownerEmail: string, orgId: string | null): string {
+  return orgId ? `org:${orgId}` : `user:${ownerEmail}`;
+}
+
+function compositeKey(parts: readonly string[]): string {
+  return JSON.stringify(parts);
+}
+
+function stableId(prefix: string, parts: readonly string[]): string {
+  return `${prefix}_${parts.map((part) => encodeURIComponent(part)).join("|")}`;
+}
+
+/**
+ * Upsert compact rollups after the corresponding raw analytics rows commit.
+ * The caller owns the raw insert; this function deliberately does not read or
+ * write analytics_events, so ingestion can choose its own failure boundary.
+ */
+export async function upsertFirstPartyAnalyticsRollups(
+  rows: readonly NormalizedFirstPartyAnalyticsEventRow[],
+): Promise<FirstPartyAnalyticsRollupResult> {
+  const dailyRollups = new Map<string, DailyRollupRow>();
+  const userDays = new Map<string, UserDayRow>();
+
+  for (const row of rows) {
+    const eventName = requiredString(row.eventName, "eventName");
+    const eventDate = requiredString(row.eventDate, "eventDate");
+    const ownerEmail = requiredString(row.ownerEmail, "ownerEmail");
+    const orgId = nullableString(row.orgId, "orgId");
+    const userKey = nullableString(row.userKey, "userKey");
+    const app = nullableString(row.app, "app") ?? "";
+    const template = nullableString(row.template, "template") ?? "";
+    const scopeKey = tenantKey(ownerEmail, orgId);
+    const dailyKey = compositeKey([
+      scopeKey,
+      eventDate,
+      eventName,
+      app,
+      template,
+    ]);
+    const existingDaily = dailyRollups.get(dailyKey);
+
+    if (existingDaily) {
+      existingDaily.eventCount += 1;
+    } else {
+      dailyRollups.set(dailyKey, {
+        id: stableId("aedr", [scopeKey, eventDate, eventName, app, template]),
+        tenantKey: scopeKey,
+        ownerEmail,
+        orgId,
+        eventDate,
+        eventName,
+        app,
+        template,
+        eventCount: 1,
+      });
+    }
+
+    if (userKey) {
+      const userDayKey = compositeKey([scopeKey, eventDate, userKey]);
+      if (!userDays.has(userDayKey)) {
+        userDays.set(userDayKey, {
+          id: stableId("aud", [scopeKey, eventDate, userKey]),
+          tenantKey: scopeKey,
+          ownerEmail,
+          orgId,
+          eventDate,
+          userKey,
+        });
+      }
+    }
+  }
+
+  if (dailyRollups.size === 0) {
+    return {
+      eventCount: rows.length,
+      dailyRollupCount: 0,
+      userDayCount: 0,
+    };
+  }
+
+  const db = getDb() as any;
+  await db.transaction(async (tx: any) => {
+    const dailyRows = [...dailyRollups.values()];
+    await tx
+      .insert(schema.analyticsEventDailyRollups)
+      .values(dailyRows)
+      .onConflictDoUpdate({
+        target: [
+          schema.analyticsEventDailyRollups.tenantKey,
+          schema.analyticsEventDailyRollups.eventDate,
+          schema.analyticsEventDailyRollups.eventName,
+          schema.analyticsEventDailyRollups.app,
+          schema.analyticsEventDailyRollups.template,
+        ],
+        set: {
+          eventCount: sql`${schema.analyticsEventDailyRollups.eventCount} + excluded.event_count`,
+        },
+      });
+
+    const userDayRows = [...userDays.values()];
+    if (userDayRows.length > 0) {
+      await tx
+        .insert(schema.analyticsUserDays)
+        .values(userDayRows)
+        .onConflictDoNothing({
+          target: [
+            schema.analyticsUserDays.tenantKey,
+            schema.analyticsUserDays.eventDate,
+            schema.analyticsUserDays.userKey,
+          ],
+        });
+    }
+  });
+
+  return {
+    eventCount: rows.length,
+    dailyRollupCount: dailyRollups.size,
+    userDayCount: userDays.size,
+  };
+}