@oneuptime/common 11.5.9 → 11.5.11

package/Models/AnalyticsModels/NetworkFlow.ts

@@ -0,0 +1,397 @@
+import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
+import Route from "../../Types/API/Route";
+import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
+import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import ObjectID from "../../Types/ObjectID";
+import Permission from "../../Types/Permission";
+
+/*
+ * One NetFlow v5 flow record exported by a network device, received by a
+ * probe's NetFlow receiver and correlated to a NetworkDevice on ingest.
+ * Powers top-talker / bandwidth-attribution queries: who talked to whom,
+ * over which protocol/port, and how many bytes/packets, per device.
+ *
+ * Access control mirrors the NetworkDevice database model — flows are an
+ * attribute of the device that exported them.
+ */
+
+const readPermissions: Array<Permission> = [
+  Permission.ProjectOwner,
+  Permission.ProjectAdmin,
+  Permission.ProjectMember,
+  Permission.Viewer,
+  Permission.SettingsAdmin,
+  Permission.SettingsMember,
+  Permission.SettingsViewer,
+  Permission.ReadNetworkDevice,
+];
+
+const createPermissions: Array<Permission> = [
+  Permission.ProjectOwner,
+  Permission.ProjectAdmin,
+  Permission.ProjectMember,
+  Permission.SettingsAdmin,
+  Permission.SettingsMember,
+  Permission.CreateNetworkDevice,
+];
+
+export default class NetworkFlow extends AnalyticsBaseModel {
+  public constructor() {
+    const projectIdColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "projectId",
+      title: "Project ID",
+      description: "ID of project",
+      required: true,
+      type: TableColumnType.ObjectID,
+      isTenantId: true,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    /*
+     * Required (not Nullable) even though a flow could in principle lack a
+     * device: the ingest path DROPS flows whose exporter matches no
+     * NetworkDevice (there is no project to attribute them to), so every
+     * stored row has one — and the column sits in the sort key, where
+     * ClickHouse rejects Nullable columns.
+     */
+    const networkDeviceIdColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "networkDeviceId",
+        title: "Network Device ID",
+        description:
+          "ID of the NetworkDevice that exported this flow (correlated from the exporter IP on ingest)",
+        required: true,
+        type: TableColumnType.ObjectID,
+        accessControl: {
+          read: readPermissions,
+          create: createPermissions,
+          update: [],
+        },
+      });
+
+    const exporterIpColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "exporterIp",
+      isLowCardinality: true,
+      title: "Exporter IP",
+      description:
+        "Source IP of the NetFlow export datagram — the router/switch that observed the flow",
+      required: true,
+      type: TableColumnType.Text,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const srcIpColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "srcIp",
+      codec: { codec: "ZSTD", level: 1 },
+      title: "Source IP",
+      description: "Source IP address of the flow's traffic",
+      required: true,
+      type: TableColumnType.Text,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const dstIpColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "dstIp",
+      codec: { codec: "ZSTD", level: 1 },
+      title: "Destination IP",
+      description: "Destination IP address of the flow's traffic",
+      required: true,
+      type: TableColumnType.Text,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const srcPortColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "srcPort",
+      title: "Source Port",
+      description: "TCP/UDP source port (0 when not applicable)",
+      required: true,
+      type: TableColumnType.Number,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const dstPortColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "dstPort",
+      title: "Destination Port",
+      description: "TCP/UDP destination port (0 when not applicable)",
+      required: true,
+      type: TableColumnType.Number,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const protocolColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "protocol",
+      title: "Protocol",
+      description: "IP protocol number (6 = TCP, 17 = UDP, 1 = ICMP, ...)",
+      required: true,
+      type: TableColumnType.Number,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const octetsColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "octets",
+      title: "Octets",
+      description: "Total bytes in the flow",
+      required: true,
+      type: TableColumnType.UInt64,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const packetsColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "packets",
+      title: "Packets",
+      description: "Total packets in the flow",
+      required: true,
+      type: TableColumnType.UInt64,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const flowStartAtColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "flowStartAt",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
+      title: "Flow Start",
+      description: "Wall-clock time the flow started",
+      required: true,
+      type: TableColumnType.DateTime64,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const flowEndAtColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "flowEndAt",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
+      title: "Flow End",
+      description: "Wall-clock time the flow ended",
+      required: true,
+      type: TableColumnType.DateTime64,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    const ingestedAtColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "ingestedAt",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
+      title: "Ingested At",
+      description: "When the server ingested this flow record",
+      required: true,
+      type: TableColumnType.DateTime64,
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [],
+      },
+    });
+
+    super({
+      tableName: AnalyticsTableName.NetworkFlow,
+      tableEngine: AnalyticsTableEngine.MergeTree,
+      singularName: "Network Flow",
+      accessControl: {
+        read: readPermissions,
+        create: createPermissions,
+        update: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.SettingsAdmin,
+          Permission.SettingsMember,
+          Permission.EditNetworkDevice,
+        ],
+        delete: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.SettingsAdmin,
+          Permission.SettingsMember,
+          Permission.DeleteNetworkDevice,
+        ],
+      },
+      pluralName: "Network Flows",
+      crudApiPath: new Route("/network-flow"),
+      tableColumns: [
+        projectIdColumn,
+        networkDeviceIdColumn,
+        exporterIpColumn,
+        srcIpColumn,
+        dstIpColumn,
+        srcPortColumn,
+        dstPortColumn,
+        protocolColumn,
+        octetsColumn,
+        packetsColumn,
+        flowStartAtColumn,
+        flowEndAtColumn,
+        ingestedAtColumn,
+      ],
+      projections: [],
+      sortKeys: ["projectId", "networkDeviceId", "flowStartAt"],
+      primaryKeys: ["projectId", "networkDeviceId", "flowStartAt"],
+      partitionKey: "toYYYYMMDD(flowStartAt)",
+      /*
+       * Shard by (projectId, networkDeviceId, flowStartAt) — mirrors how
+       * Log shards on its always-present sort-key columns: the
+       * high-entropy time component spreads even a single very busy
+       * exporter across all shards.
+       */
+      shardingKey: "cityHash64(projectId, networkDeviceId, flowStartAt)",
+      tableSettings: "non_replicated_deduplication_window = 10000",
+      /*
+       * Flows are the highest-volume rows in the product (a single busy
+       * router can export thousands per second), so the table must never
+       * grow unboundedly. Fixed-window TTL for now; per-project retention
+       * (a retentionDate column computed at ingest, like Log/Metric) is
+       * the phase-2 follow-up. Keyed on server-assigned ingestedAt so a
+       * device with a wrong clock cannot make rows expire early.
+       */
+      ttlExpression: "ingestedAt + INTERVAL 30 DAY DELETE",
+      defaultSortColumn: "flowStartAt",
+    });
+  }
+
+  public get projectId(): ObjectID | undefined {
+    return this.getColumnValue("projectId") as ObjectID | undefined;
+  }
+
+  public set projectId(v: ObjectID | undefined) {
+    this.setColumnValue("projectId", v);
+  }
+
+  public get networkDeviceId(): ObjectID | undefined {
+    return this.getColumnValue("networkDeviceId") as ObjectID | undefined;
+  }
+
+  public set networkDeviceId(v: ObjectID | undefined) {
+    this.setColumnValue("networkDeviceId", v);
+  }
+
+  public get exporterIp(): string | undefined {
+    return this.getColumnValue("exporterIp") as string | undefined;
+  }
+
+  public set exporterIp(v: string | undefined) {
+    this.setColumnValue("exporterIp", v);
+  }
+
+  public get srcIp(): string | undefined {
+    return this.getColumnValue("srcIp") as string | undefined;
+  }
+
+  public set srcIp(v: string | undefined) {
+    this.setColumnValue("srcIp", v);
+  }
+
+  public get dstIp(): string | undefined {
+    return this.getColumnValue("dstIp") as string | undefined;
+  }
+
+  public set dstIp(v: string | undefined) {
+    this.setColumnValue("dstIp", v);
+  }
+
+  public get srcPort(): number | undefined {
+    return this.getColumnValue("srcPort") as number | undefined;
+  }
+
+  public set srcPort(v: number | undefined) {
+    this.setColumnValue("srcPort", v);
+  }
+
+  public get dstPort(): number | undefined {
+    return this.getColumnValue("dstPort") as number | undefined;
+  }
+
+  public set dstPort(v: number | undefined) {
+    this.setColumnValue("dstPort", v);
+  }
+
+  public get protocol(): number | undefined {
+    return this.getColumnValue("protocol") as number | undefined;
+  }
+
+  public set protocol(v: number | undefined) {
+    this.setColumnValue("protocol", v);
+  }
+
+  public get octets(): number | undefined {
+    return this.getColumnValue("octets") as number | undefined;
+  }
+
+  public set octets(v: number | undefined) {
+    this.setColumnValue("octets", v);
+  }
+
+  public get packets(): number | undefined {
+    return this.getColumnValue("packets") as number | undefined;
+  }
+
+  public set packets(v: number | undefined) {
+    this.setColumnValue("packets", v);
+  }
+
+  public get flowStartAt(): Date | undefined {
+    return this.getColumnValue("flowStartAt") as Date | undefined;
+  }
+
+  public set flowStartAt(v: Date | undefined) {
+    this.setColumnValue("flowStartAt", v);
+  }
+
+  public get flowEndAt(): Date | undefined {
+    return this.getColumnValue("flowEndAt") as Date | undefined;
+  }
+
+  public set flowEndAt(v: Date | undefined) {
+    this.setColumnValue("flowEndAt", v);
+  }
+
+  public get ingestedAt(): Date | undefined {
+    return this.getColumnValue("ingestedAt") as Date | undefined;
+  }
+
+  public set ingestedAt(v: Date | undefined) {
+    this.setColumnValue("ingestedAt", v);
+  }
+}

package/Models/DatabaseModels/EnterpriseLicenseInstance.ts

@@ -170,4 +170,22 @@ export default class EnterpriseLicenseInstance extends BaseModel {
     type: ColumnType.Date,
   })
   public lastReportedAt?: Date = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    required: false,
+    type: TableColumnType.JSON,
+    title: "Master Admin Emails",
+    description:
+      "Emails of the master admins of this instance. Used to contact the customer about license expiry and seat-limit breaches.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: true,
+  })
+  public masterAdminEmails?: Array<string> = undefined;
 }

package/Models/DatabaseModels/GlobalConfig.ts

@@ -893,6 +893,42 @@ export default class GlobalConfig extends GlobalConfigModel {
   public enterpriseLicenseInstances?: Array<EnterpriseLicenseInstanceSummary> =
     undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.Email,
+    title: "Enterprise License Notification Email",
+    description:
+      "OneUptime email address CC'd on enterprise license notifications sent to customers (expiry reminders, seat-limit breaches). Only used on the hosted oneuptime.com (billing enabled).",
+  })
+  @Column({
+    type: ColumnType.Email,
+    length: ColumnLength.Email,
+    nullable: true,
+    transformer: Email.getDatabaseTransformer(),
+  })
+  public enterpriseLicenseNotificationEmail?: Email = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.Number,
+    title: "Enterprise License Expiry Reminder Days",
+    description:
+      "How many days before an enterprise license expires that expiry reminder emails start going out to the customer's master admins. Defaults to 45. Only used on the hosted oneuptime.com (billing enabled).",
+  })
+  @Column({
+    type: ColumnType.Number,
+    nullable: true,
+  })
+  public enterpriseLicenseExpiryReminderDays?: number = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [],

package/Models/DatabaseModels/Index.ts

@@ -401,6 +401,7 @@ import MetricType from "./MetricType";
 import ProjectSCIM from "./ProjectSCIM";
 import ProjectSCIMLog from "./ProjectSCIMLog";
 import StatusPageSCIMLog from "./StatusPageSCIMLog";
+import MarketingConversion from "./MarketingConversion";
 
 const AllModelTypes: Array<{
   new (): BaseModel;
@@ -824,6 +825,7 @@ const AllModelTypes: Array<{
   ServerlessFunctionInstance,
   CloudResourceInstance,
   RumApplicationClient,
+  MarketingConversion,
 ];
 
 const modelTypeMap: { [key: string]: { new (): BaseModel } } = {};

package/Models/DatabaseModels/MarketingConversion.ts

@@ -0,0 +1,198 @@
+import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
+import Route from "../../Types/API/Route";
+import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
+import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
+import ColumnLength from "../../Types/Database/ColumnLength";
+import ColumnType from "../../Types/Database/ColumnType";
+import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
+import TableColumn from "../../Types/Database/TableColumn";
+import TableColumnType from "../../Types/Database/TableColumnType";
+import TableMetadata from "../../Types/Database/TableMetadata";
+import IconProp from "../../Types/Icon/IconProp";
+import { JSONObject } from "../../Types/JSON";
+import ObjectID from "../../Types/ObjectID";
+import { Column, Entity, Index } from "typeorm";
+
+/*
+ * Internal table (no API access) recording ad-attributed conversions —
+ * signups and paid subscriptions from visitors that carried ad click IDs —
+ * and the status of uploading each of them to ad platforms (Google Ads
+ * offline click conversions, Meta Conversions API). Written and read only by
+ * the MarketingConversions worker job.
+ */
+@TableAccessControl({
+  create: [],
+  read: [],
+  update: [],
+  delete: [],
+})
+@CrudApiEndpoint(new Route("/marketing-conversion"))
+@TableMetadata({
+  tableName: "MarketingConversion",
+  singularName: "Marketing Conversion",
+  pluralName: "Marketing Conversions",
+  icon: IconProp.ChartBar,
+  tableDescription:
+    "Ad-attributed conversions (signups, paid subscriptions) and their upload status to ad platforms for offline conversion tracking.",
+})
+@Entity({
+  name: "MarketingConversion",
+})
+@Index("uq_marketing_conversion_type_user", ["conversionType", "userId"], {
+  unique: true,
+})
+@Index(
+  "uq_marketing_conversion_type_project",
+  ["conversionType", "projectId"],
+  {
+    unique: true,
+  },
+)
+export default class MarketingConversion extends BaseModel {
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.ShortText,
+    required: true,
+    title: "Conversion Type",
+    description: "SignUp or PaidSubscription.",
+  })
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    nullable: false,
+  })
+  public conversionType?: string = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @Index("idx_marketing_conversion_user_id")
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    title: "User ID",
+    description: "User this conversion belongs to (SignUp conversions).",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public userId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @Index("idx_marketing_conversion_project_id")
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    title: "Project ID",
+    description:
+      "Project this conversion belongs to (PaidSubscription conversions).",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public projectId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.ShortText,
+    required: false,
+    title: "Email",
+    description:
+      "Email of the converting user. Hashed before being sent to ad platforms that support enhanced matching (Meta).",
+  })
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    nullable: true,
+  })
+  public email?: string | undefined = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.JSON,
+    required: true,
+    title: "Click IDs",
+    description:
+      "Ad click identifiers (gclid, wbraid, gbraid, fbclid, ...) captured for this conversion.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: false,
+  })
+  public clickIds?: JSONObject = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.Date,
+    required: true,
+    title: "Conversion At",
+    description: "When the conversion happened.",
+  })
+  @Column({
+    type: ColumnType.Date,
+    nullable: false,
+  })
+  public conversionAt?: Date = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.Number,
+    required: false,
+    title: "Conversion Value (USD Cents)",
+    description:
+      "Conversion value in USD cents (monthly recurring revenue for paid subscriptions). Null when unknown (custom pricing) or not applicable (signups).",
+  })
+  @Column({
+    type: ColumnType.Number,
+    nullable: true,
+  })
+  public conversionValueInUSDCents?: number | undefined = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.JSON,
+    required: false,
+    title: "Upload State",
+    description:
+      "Per-ad-platform upload state, keyed by provider (google, meta, microsoft, linkedin, reddit, ...): { status: Uploaded|Failed|Skipped, attempts, error, uploadedAt }. Absent key or absent status means pending.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: true,
+  })
+  public uploadState?: JSONObject = undefined;
+}