@l.x/datadog-cloud 1.0.0

package/index.ts

@@ -0,0 +1,79 @@
+import * as pulumi from '@pulumi/pulumi';
+import {createMonitors} from './factory';
+import {
+  devPortalLatencyMonitors,
+  devPortalErrorMonitors,
+  devPortalAvailabilityMonitors,
+  devPortalAuthMonitors,
+  devPortalGatewayMonitors,
+  devPortalLogMonitors,
+} from './monitors';
+import {settings} from './config';
+import {MonitorDefinition} from './types';
+
+// Log configuration
+pulumi.log.info(`Environment: ${settings.environment}`);
+pulumi.log.info(`Team: ${settings.defaultTeam}`);
+pulumi.log.info(`Tag Filter: ${settings.tagFilter}`);
+
+const isProd = settings.environment === 'prod';
+
+// Create monitors based on the team specified in config
+// Each stack should only create monitors for its team
+const team = settings.defaultTeam;
+
+// Team monitor definitions
+const teamMonitors: Record<
+  string,
+  {monitors: MonitorDefinition[]; category: string}[]
+> = {
+  'dev-portal': [
+    {monitors: devPortalLatencyMonitors, category: 'latency'},
+    {monitors: devPortalErrorMonitors, category: 'errors'},
+    {monitors: devPortalAvailabilityMonitors, category: 'availability'},
+    {monitors: devPortalAuthMonitors, category: 'auth'},
+    {monitors: devPortalGatewayMonitors, category: 'gateway'},
+    {monitors: devPortalLogMonitors, category: 'logs'},
+  ],
+};
+
+// Get monitors for current team
+const currentTeamMonitors = teamMonitors[team];
+if (!currentTeamMonitors) {
+  throw new Error(
+    `Unknown team: ${team}. Valid teams: ${Object.keys(teamMonitors).join(', ')}`
+  );
+}
+
+// Create monitors for current team
+const createdMonitors: Record<
+  string,
+  Record<string, pulumi.Output<string>>
+> = {};
+let totalCount = 0;
+
+for (const {monitors, category} of currentTeamMonitors) {
+  // Filter out prodOnly monitors on non-prod stacks
+  const filtered = isProd ? monitors : monitors.filter(m => !m.prodOnly);
+  const created = createMonitors(filtered);
+  createdMonitors[category] = Object.fromEntries(
+    Object.entries(created).map(([k, v]) => [k, v.id])
+  );
+  totalCount += Object.keys(created).length;
+}
+
+// Export monitor IDs grouped by category
+export const monitorIds = createdMonitors;
+
+// Export summary
+export const summary = {
+  team,
+  environment: settings.environment,
+  totalMonitors: totalCount,
+  byCategory: Object.fromEntries(
+    Object.entries(createdMonitors).map(([cat, monitors]) => [
+      cat,
+      Object.keys(monitors).length,
+    ])
+  ),
+};

package/monitors/dev-portal/auth.ts

@@ -0,0 +1,44 @@
+import {MonitorDefinition} from '../../types';
+
+export const devPortalAuthMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_auth_failure_rate',
+    name: 'Auth Failure Rate on dev-portal',
+    type: 'log alert',
+    query: `logs("service:dev-portal (message:*auth*failed OR message:*session*failed)").index("*").rollup("count").last("15m") > 20`,
+    alertBody:
+      'Auth failure log count for dev-portal has exceeded 20 in the last 15 minutes.',
+    recoveryBody: 'Auth failure rate for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 3,
+    thresholds: {critical: 20},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    enablePaging: false,
+    onMissingData: 'show_no_data',
+    prodOnly: true,
+  },
+  {
+    id: 'dev-portal_session_conflict_spike',
+    name: 'Session Conflict Spike on dev-portal',
+    type: 'log alert',
+    query: `logs("service:dev-portal message:*conflict").index("*").rollup("count").last("15m") > 10`,
+    alertBody:
+      'Session conflict log count for dev-portal has exceeded 10 in the last 15 minutes.',
+    recoveryBody: 'Session conflict rate for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 3,
+    thresholds: {critical: 10},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    enablePaging: false,
+    onMissingData: 'show_no_data',
+    prodOnly: true,
+  },
+];

package/monitors/dev-portal/availability.ts

@@ -0,0 +1,44 @@
+import {MonitorDefinition} from '../../types';
+import {settings} from '../../config';
+
+// ALB metrics use name:dev-portal-lb tag, scoped by environment from Pulumi config
+const albTagFilter = `name:dev-portal-lb,unienv:${settings.environment}`;
+
+export const devPortalAvailabilityMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_availability_zero_traffic',
+    name: 'Zero Traffic on dev-portal',
+    type: 'query alert',
+    query: `sum(last_10m):sum:aws.applicationelb.request_count{${albTagFilter}}.as_count() == 0`,
+    alertBody: 'dev-portal is receiving zero traffic. Service may be down.',
+    recoveryBody: 'dev-portal is receiving traffic again.',
+    team: 'dev-portal',
+    priority: 1,
+    thresholds: {critical: 0},
+    noDataTimeframe: 15,
+    notifyNoData: true,
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+  },
+  {
+    id: 'dev-portal_availability_success_rate',
+    name: 'Success Rate on dev-portal',
+    type: 'query alert',
+    query: `sum(last_5m):(1 - sum:aws.applicationelb.httpcode_target_5xx{${albTagFilter}}.as_count() / sum:aws.applicationelb.request_count{${albTagFilter}}.as_count()) * 100 < 99`,
+    alertBody: 'Success rate for dev-portal has dropped below 99%.',
+    recoveryBody: 'Success rate for dev-portal has recovered above 99%.',
+    team: 'dev-portal',
+    priority: 2,
+    thresholds: {critical: 99, warning: 99.5},
+    logQuery: 'service:dev-portal status:error',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    notifyNoData: true,
+    noDataTimeframe: 15,
+  },
+];

package/monitors/dev-portal/errors.ts

@@ -0,0 +1,47 @@
+import {MonitorDefinition} from '../../types';
+import {settings} from '../../config';
+
+// ALB metrics use name:dev-portal-lb tag, scoped by environment from Pulumi config
+const albTagFilter = `name:dev-portal-lb,unienv:${settings.environment}`;
+
+export const devPortalErrorMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_error_5xx',
+    name: '5xx Error Rate on dev-portal',
+    type: 'query alert',
+    query: `sum(last_5m):sum:aws.applicationelb.httpcode_target_5xx{${albTagFilter}}.as_count() / sum:aws.applicationelb.request_count{${albTagFilter}}.as_count() * 100 > 5`,
+    alertBody: '5xx error rate for dev-portal has exceeded 5%.',
+    recoveryBody: '5xx error rate for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 2,
+    thresholds: {critical: 5, warning: 2},
+    logQuery: 'service:dev-portal status:error',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    notifyNoData: true,
+    noDataTimeframe: 15,
+  },
+  {
+    id: 'dev-portal_error_anomaly',
+    name: 'Error Count Anomaly on dev-portal',
+    type: 'query alert',
+    query: `avg(last_4h):anomalies(sum:aws.applicationelb.httpcode_target_5xx{${albTagFilter}}.as_count(), 'agile', 3, direction='above', interval=300, alert_window='last_30m', count_default_zero='true') >= 1`,
+    alertBody: 'Anomalous spike in error count detected for dev-portal.',
+    recoveryBody: 'Error count anomaly for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 3,
+    thresholds: {critical: 1, warning: 0.8},
+    thresholdWindows: {
+      triggerWindow: 'last_30m',
+      recoveryWindow: 'last_30m',
+    },
+    logQuery: 'service:dev-portal status:error',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    enablePaging: false,
+  },
+];

package/monitors/dev-portal/gateway.ts

@@ -0,0 +1,47 @@
+import {MonitorDefinition} from '../../types';
+
+export const devPortalGatewayMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_gateway_proxy_errors',
+    name: 'Gateway Proxy Errors on dev-portal',
+    type: 'log alert',
+    query:
+      'logs("service:dev-portal message:gateway.proxy.error").index("*").rollup("count").last("10m") > 5',
+    alertBody:
+      'Gateway proxy errors on dev-portal have exceeded threshold. The app is logging gateway.proxy.error at error level — check url, method, and duration_ms fields for details.',
+    recoveryBody:
+      'Gateway proxy errors on dev-portal have recovered.',
+    team: 'dev-portal',
+    priority: 2,
+    thresholds: {critical: 5},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    onMissingData: 'show_no_data',
+    prodOnly: true,
+  },
+  {
+    id: 'dev-portal_gateway_proxy_high_latency',
+    name: 'Gateway Proxy High Latency on dev-portal',
+    type: 'log alert',
+    query:
+      'logs("service:dev-portal message:gateway.proxy.complete @duration_ms:>3000").index("*").rollup("count").last("10m") > 10',
+    alertBody:
+      'Gateway proxy latency on dev-portal is consistently high. More than 10 requests exceeded 3000ms duration in the last 10 minutes.',
+    recoveryBody:
+      'Gateway proxy latency on dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 3,
+    thresholds: {critical: 10},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    enablePaging: false,
+    onMissingData: 'show_no_data',
+    prodOnly: true,
+  },
+];

package/monitors/dev-portal/index.ts

@@ -0,0 +1,6 @@
+export {devPortalLatencyMonitors} from './latency';
+export {devPortalErrorMonitors} from './errors';
+export {devPortalAvailabilityMonitors} from './availability';
+export {devPortalAuthMonitors} from './auth';
+export {devPortalGatewayMonitors} from './gateway';
+export {devPortalLogMonitors} from './logs';

package/monitors/dev-portal/latency.ts

@@ -0,0 +1,44 @@
+import {MonitorDefinition} from '../../types';
+import {settings} from '../../config';
+
+// ALB metrics use name:dev-portal-lb tag, scoped by environment from Pulumi config
+const albTagFilter = `name:dev-portal-lb,unienv:${settings.environment}`;
+
+export const devPortalLatencyMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_latency_p95',
+    name: 'P95 Latency on dev-portal',
+    type: 'query alert',
+    query: `avg(last_5m):avg:aws.applicationelb.target_response_time.p95{${albTagFilter}} > 2`,
+    alertBody: 'P95 latency for dev-portal has exceeded 2 seconds.',
+    recoveryBody: 'P95 latency for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 3,
+    thresholds: {critical: 2, warning: 1.5},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    notifyNoData: true,
+    noDataTimeframe: 15,
+  },
+  {
+    id: 'dev-portal_latency_p99',
+    name: 'P99 Latency on dev-portal',
+    type: 'query alert',
+    query: `avg(last_5m):avg:aws.applicationelb.target_response_time.p99{${albTagFilter}} > 5`,
+    alertBody: 'P99 latency for dev-portal has exceeded 5 seconds.',
+    recoveryBody: 'P99 latency for dev-portal has recovered.',
+    team: 'dev-portal',
+    priority: 2,
+    thresholds: {critical: 5, warning: 3},
+    logQuery: 'service:dev-portal',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    notifyNoData: true,
+    noDataTimeframe: 15,
+  },
+];

package/monitors/dev-portal/logs.ts

@@ -0,0 +1,26 @@
+import {MonitorDefinition} from '../../types';
+
+export const devPortalLogMonitors: MonitorDefinition[] = [
+  {
+    id: 'dev-portal_log_error_spike',
+    name: 'Error Log Spike on dev-portal',
+    type: 'log alert',
+    query:
+      'logs("service:dev-portal level:error").index("*").rollup("count").last("15m") > 50',
+    alertBody:
+      'Error-level log volume for dev-portal has exceeded 50 in the last 15 minutes.',
+    recoveryBody:
+      'Error-level log volume for dev-portal has recovered to normal levels.',
+    team: 'dev-portal',
+    priority: 2,
+    thresholds: {critical: 50},
+    logQuery: 'service:dev-portal level:error',
+    runbookUrl: 'https://www.notion.so/lx/dev-portal-runbook',
+    readmeUrl:
+      'https://github.com/Lx/universe/tree/main/apps/dev-portal',
+    dashboards: [],
+    enablePaging: false,
+    onMissingData: 'show_no_data',
+    prodOnly: true,
+  },
+];

package/monitors/index.ts

@@ -0,0 +1,8 @@
+export {
+  devPortalLatencyMonitors,
+  devPortalErrorMonitors,
+  devPortalAvailabilityMonitors,
+  devPortalAuthMonitors,
+  devPortalGatewayMonitors,
+  devPortalLogMonitors,
+} from './dev-portal';

package/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@l.x/datadog-cloud",
+  "version": "1.0.0",
+  "description": "Datadog monitors and dashboards for Universe services",
+  "main": "index.ts",
+  "dependencies": {
+    "@pulumi/datadog": "4.60.0",
+    "@pulumi/pulumi": "3.207.0"
+  },
+  "devDependencies": {
+    "@types/node": "22.13.1",
+    "typescript": "5.8.3"
+  },
+  "scripts": {
+    "preview": "pulumi preview",
+    "up": "pulumi up",
+    "destroy": "pulumi destroy",
+    "refresh": "pulumi refresh"
+  }
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "moduleResolution": "node",
+    "outDir": "./bin",
+    "rootDir": ".",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["./**/*.ts"],
+  "exclude": ["node_modules", "bin", "dashboards"]
+}

package/types.ts

@@ -0,0 +1,129 @@
+import * as datadog from '@pulumi/datadog';
+
+/**
+ * Dashboard link configuration
+ */
+export interface DashboardLink {
+  /** Display name for the dashboard */
+  name: string;
+  /** Datadog dashboard URL path (e.g., /dashboard/abc-123) */
+  url: string;
+}
+
+/**
+ * Monitor definition input - simplified interface for defining monitors
+ */
+export interface MonitorDefinition {
+  /** Unique identifier for the monitor (used in resource name and tags) */
+  id: string;
+
+  /** Display name - can include template variables like {{service.name}} */
+  name: string;
+
+  /** Monitor type */
+  type:
+    | 'query alert'
+    | 'event-v2 alert'
+    | 'composite'
+    | 'log alert'
+    | 'metric alert'
+    | 'ci-pipelines alert';
+
+  /** Datadog query */
+  query: string;
+
+  /** Alert message body (will be wrapped with standardized links section) */
+  alertBody: string;
+
+  /** Recovery message body (optional) */
+  recoveryBody?: string;
+
+  /** Team that owns this monitor - used to lookup EP from ESC */
+  team: string;
+
+  /** Monitor priority (1-5, where 1 is highest) */
+  priority: 1 | 2 | 3 | 4 | 5;
+
+  /**
+   * Log query filter for the affected resource.
+   * Used to generate log links in alert body.
+   * Examples:
+   *   - "service:{{service.name}}"
+   *   - "host:{{host.name}}"
+   */
+  logQuery: string;
+
+  /**
+   * Notion runbook URL for this monitor/service.
+   * Should contain troubleshooting steps and escalation procedures.
+   */
+  runbookUrl: string;
+
+  /**
+   * GitHub README URL for the service codebase.
+   * Links to the service's documentation in the repository.
+   */
+  readmeUrl: string;
+
+  /**
+   * Relevant Datadog dashboards for investigating this alert.
+   */
+  dashboards: DashboardLink[];
+
+  /** Threshold configuration */
+  thresholds?: {
+    critical?: number;
+    warning?: number;
+    criticalRecovery?: number;
+    warningRecovery?: number;
+  };
+
+  /** Threshold windows for anomaly monitors */
+  thresholdWindows?: {
+    triggerWindow?: string;
+    recoveryWindow?: string;
+  };
+
+  /** Minutes before notifying on missing data (0 = disabled) */
+  noDataTimeframe?: number;
+
+  /** Notify when data stops reporting */
+  notifyNoData?: boolean;
+
+  /** Minutes between re-notifications (0 = disabled) */
+  renotifyInterval?: number;
+
+  /** Seconds to wait before evaluating new groups */
+  newGroupDelay?: number;
+
+  /** Seconds to delay metric evaluation */
+  evaluationDelay?: number;
+
+  /** Additional tags beyond the standard ones */
+  additionalTags?: string[];
+
+  /** Include incident.io webhook (default: true) */
+  includeIncidentWebhook?: boolean;
+
+  /** Enable paging via escalation policy tag (default: true) */
+  enablePaging?: boolean;
+
+  /** Only deploy this monitor on prod stacks (for environment-agnostic queries that would duplicate) */
+  prodOnly?: boolean;
+
+  /** On missing data behavior */
+  onMissingData?:
+    | 'default'
+    | 'show_no_data'
+    | 'resolve'
+    | 'show_and_notify_no_data';
+}
+
+/**
+ * Standard monitor options
+ */
+export const defaultMonitorOptions: Partial<datadog.MonitorArgs> = {
+  notifyAudit: false,
+  includeTags: false,
+  onMissingData: 'default',
+};