@oneuptime/common 11.7.4 → 12.0.1

package/build/dist/Models/AnalyticsModels/RumSessionChunk.js

@@ -0,0 +1,670 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
+import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
+import AnalyticsTableColumn, { SkipIndexType, } from "../../Types/AnalyticsDatabase/TableColumn";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import OperationalResource from "../../Types/Database/AccessControl/OperationalResource";
+import OwnedThrough from "../../Types/Database/AccessControl/OwnedThrough";
+import Permission from "../../Types/Permission";
+import { PlanType } from "../../Types/Billing/SubscriptionPlan";
+import RumApplication from "../DatabaseModels/RumApplication";
+/*
+ * The recorded payload for session replay — one row per ~15s frame.
+ *
+ * This is the fattest table in the system and the most sensitive: a row's
+ * `payload` column is a serialised recording of what a real person had on
+ * screen. Several deliberate departures from the house pattern follow, all
+ * because of that.
+ *
+ * The only read that exists is
+ *   WHERE projectId = ? AND sessionId = ? AND tabId = ? AND chunkIndex IN (...)
+ * so the sort key carries no timestamp at all. A time-first key would
+ * scatter one session's dozen rows across a whole day of marks.
+ */
+/*
+ * Chunk *metadata* is readable by anyone who may list sessions — it is
+ * what draws the timeline. The payload itself is not; see below.
+ */
+const CHUNK_METADATA_PERMISSIONS = [
+    Permission.ProjectOwner,
+    Permission.ProjectAdmin,
+    Permission.TelemetryAdmin,
+    Permission.ReadRumSessionReplay,
+];
+const CHUNK_CREATE_PERMISSIONS = [
+    Permission.ProjectOwner,
+    Permission.ProjectAdmin,
+    Permission.TelemetryAdmin,
+    Permission.CreateRumSessionReplay,
+];
+const chunkAccessControl = {
+    read: CHUNK_METADATA_PERMISSIONS,
+    create: CHUNK_CREATE_PERMISSIONS,
+    update: [],
+};
+/*
+ * Watching a recording is its own privilege, deliberately narrower than
+ * listing sessions and narrower still than reading the RUM application.
+ *
+ * Note this column ACL is defence in depth, NOT the primary control: with
+ * crudApiPath omitted, the only reader is a bespoke endpoint that never
+ * invokes ModelPermission. The handler-level guard is the real gate.
+ */
+const payloadAccessControl = {
+    read: [
+        Permission.ProjectOwner,
+        Permission.ProjectAdmin,
+        Permission.TelemetryAdmin,
+        Permission.ReadRumSessionReplayPayload,
+    ],
+    create: CHUNK_CREATE_PERMISSIONS,
+    update: [],
+};
+let RumSessionChunk = class RumSessionChunk extends AnalyticsBaseModel {
+    constructor() {
+        const projectIdColumn = new AnalyticsTableColumn({
+            key: "projectId",
+            title: "Project ID",
+            description: "ID of project",
+            required: true,
+            type: TableColumnType.ObjectID,
+            isTenantId: true,
+            accessControl: chunkAccessControl,
+        });
+        /*
+         * Sort-key element 2, so an equality predicate on it is served by the
+         * primary index. No skip index — one would only inflate the part.
+         */
+        const sessionIdColumn = new AnalyticsTableColumn({
+            key: "sessionId",
+            title: "Session ID",
+            description: "Session this chunk belongs to",
+            required: true,
+            type: TableColumnType.Text,
+            codec: { codec: "ZSTD", level: 1 },
+            accessControl: chunkAccessControl,
+        });
+        /*
+         * In the sort key from the first commit, and not addable later.
+         *
+         * sessionStorage is COPIED when a tab is duplicated, so two live tabs
+         * can share one sessionId and both mint chunkIndex from 0. Without
+         * tabId the sort key cannot represent both streams and read-time
+         * dedup silently discards one tab's entire recording.
+         */
+        const tabIdColumn = new AnalyticsTableColumn({
+            key: "tabId",
+            title: "Tab ID",
+            description: "Per-tab discriminator, so concurrent duplicated tabs do not collide on chunkIndex",
+            required: true,
+            type: TableColumnType.Text,
+            codec: { codec: "ZSTD", level: 1 },
+            accessControl: chunkAccessControl,
+        });
+        const chunkIndexColumn = new AnalyticsTableColumn({
+            key: "chunkIndex",
+            title: "Chunk Index",
+            description: "Monotonic frame index within the (session, tab)",
+            required: true,
+            type: TableColumnType.Number,
+            codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        /* ReplacingMergeTree version. Unix milliseconds — see RumSession. */
+        const versionColumn = new AnalyticsTableColumn({
+            key: "version",
+            title: "Version",
+            description: "Write version in unix milliseconds. A redelivered chunk collapses at merge rather than double-inserting.",
+            required: true,
+            type: TableColumnType.UInt64,
+            codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        const rumApplicationIdColumn = new AnalyticsTableColumn({
+            key: "rumApplicationId",
+            title: "RUM Application ID",
+            description: "RUM application this chunk belongs to",
+            required: true,
+            type: TableColumnType.ObjectID,
+            skipIndex: {
+                name: "idx_chunk_rum_application_id",
+                type: SkipIndexType.Set,
+                params: [512],
+                granularity: 4,
+            },
+            accessControl: chunkAccessControl,
+        });
+        const primaryEntityIdColumn = new AnalyticsTableColumn({
+            key: "primaryEntityId",
+            title: "Primary Entity ID",
+            description: "Owning resource — always the RUM application id",
+            required: true,
+            type: TableColumnType.ObjectID,
+            skipIndex: {
+                name: "idx_chunk_primary_entity_id",
+                type: SkipIndexType.Set,
+                params: [512],
+                granularity: 4,
+            },
+            accessControl: chunkAccessControl,
+        });
+        const primaryEntityTypeColumn = new AnalyticsTableColumn({
+            key: "primaryEntityType",
+            title: "Primary Entity Type",
+            description: "Always RealUserMonitor",
+            required: true,
+            isLowCardinality: true,
+            type: TableColumnType.Text,
+            skipIndex: {
+                name: "idx_chunk_primary_entity_type",
+                type: SkipIndexType.Set,
+                params: [16],
+                granularity: 4,
+            },
+            accessControl: chunkAccessControl,
+        });
+        /* Identical on every chunk of a session — server-clamped, never client. */
+        const sessionStartTimeColumn = new AnalyticsTableColumn({
+            key: "sessionStartTime",
+            title: "Session Start Time",
+            description: "Server-clamped session start, identical across every chunk of a session",
+            required: true,
+            type: TableColumnType.DateTime64,
+            codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        const offsetColumns = [
+            {
+                key: "chunkStartOffsetMs",
+                title: "Chunk Start Offset (ms)",
+                description: "Offset of this chunk's first event from session start",
+            },
+            {
+                key: "chunkEndOffsetMs",
+                title: "Chunk End Offset (ms)",
+                description: "Offset of this chunk's last event from session start",
+            },
+        ].map((def) => {
+            return new AnalyticsTableColumn({
+                key: def.key,
+                title: def.title,
+                description: def.description,
+                required: true,
+                type: TableColumnType.Number,
+                codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+                accessControl: chunkAccessControl,
+            });
+        });
+        const timeColumns = [
+            {
+                key: "chunkStartTime",
+                title: "Chunk Start Time",
+                description: "Absolute time of this chunk's first event",
+            },
+            {
+                key: "chunkEndTime",
+                title: "Chunk End Time",
+                description: "Absolute time of this chunk's last event",
+            },
+        ].map((def) => {
+            return new AnalyticsTableColumn({
+                key: def.key,
+                title: def.title,
+                description: def.description,
+                required: true,
+                type: TableColumnType.DateTime64,
+                codec: { codec: "ZSTD", level: 1 },
+                accessControl: chunkAccessControl,
+            });
+        });
+        const eventCountColumn = new AnalyticsTableColumn({
+            key: "eventCount",
+            title: "Event Count",
+            description: "Recorded events in this chunk",
+            required: true,
+            type: TableColumnType.Number,
+            codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        /*
+         * Marks this chunk as a valid seek anchor. Set from rrweb's own
+         * isCheckout flag rather than inferred, and — when a large snapshot is
+         * split across parts — only on the final part, so a seek anchor never
+         * points into the middle of a snapshot.
+         */
+        const hasFullSnapshotColumn = new AnalyticsTableColumn({
+            key: "hasFullSnapshot",
+            title: "Has Full Snapshot",
+            description: "Chunk opens on a complete DOM snapshot and is therefore a valid seek anchor",
+            required: true,
+            type: TableColumnType.Boolean,
+            skipIndex: {
+                name: "idx_has_full_snapshot",
+                type: SkipIndexType.Set,
+                params: [2],
+                granularity: 4,
+            },
+            accessControl: chunkAccessControl,
+        });
+        const isFinalColumn = new AnalyticsTableColumn({
+            key: "isFinal",
+            title: "Is Final",
+            description: "Recorder marked this as the last chunk of the session",
+            required: true,
+            type: TableColumnType.Boolean,
+            accessControl: chunkAccessControl,
+        });
+        const isPinnedCopyColumn = new AnalyticsTableColumn({
+            key: "isPinnedCopy",
+            title: "Is Pinned Copy",
+            description: "Written by the pin materializer: a re-insert of this chunk with a far-future retentionDate, landing in its own retention partition so a pinned recording survives while the original partition is TTL-dropped. Same sessionId, so erasure still catches it.",
+            required: true,
+            type: TableColumnType.Boolean,
+            accessControl: chunkAccessControl,
+        });
+        const snapshotPartColumns = [
+            {
+                key: "snapshotPartIndex",
+                title: "Snapshot Part Index",
+                description: "Part index when a full snapshot is split across chunks",
+            },
+            {
+                key: "snapshotPartTotal",
+                title: "Snapshot Part Total",
+                description: "Total parts a split full snapshot was divided into",
+            },
+        ].map((def) => {
+            return new AnalyticsTableColumn({
+                key: def.key,
+                title: def.title,
+                description: def.description,
+                required: true,
+                defaultValue: 0,
+                type: TableColumnType.UInt8,
+                codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+                accessControl: chunkAccessControl,
+            });
+        });
+        const descriptorColumns = [
+            {
+                key: "recorderKind",
+                title: "Recorder Kind",
+                description: "dom for web, rn-view-tree for mobile",
+            },
+            {
+                key: "payloadEncoding",
+                title: "Payload Encoding",
+                description: "How the stored payload is encoded. Stored decompressed, so this records what arrived on the wire.",
+            },
+        ].map((def) => {
+            return new AnalyticsTableColumn({
+                key: def.key,
+                title: def.title,
+                description: def.description,
+                required: true,
+                defaultValue: "",
+                isLowCardinality: true,
+                type: TableColumnType.Text,
+                codec: { codec: "ZSTD", level: 1 },
+                accessControl: chunkAccessControl,
+            });
+        });
+        const schemaVersionColumn = new AnalyticsTableColumn({
+            key: "schemaVersion",
+            title: "Schema Version",
+            description: "Payload schema version, so a player can refuse gracefully instead of rendering a wrong DOM",
+            required: true,
+            type: TableColumnType.UInt8,
+            codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        /*
+         * The recording itself.
+         *
+         * Stored DECOMPRESSED as JSON text under ZSTD(3), not as base64 of the
+         * gzip the recorder sent: base64 inflates 33% and hands ZSTD
+         * incompressible input, so the naive approach is both bigger on disk
+         * and more CPU. There is no binary column type in this layer, so Text
+         * is the only option.
+         *
+         * No skip index of any kind: chunks are fetched by key, and an index
+         * over multi-hundred-KB values would only inflate the part.
+         */
+        const payloadColumn = new AnalyticsTableColumn({
+            key: "payload",
+            title: "Payload",
+            description: "Serialised recorded event array for this chunk. Reading this is watching a real end user's screen and is gated on its own permission.",
+            required: true,
+            type: TableColumnType.Text,
+            codec: { codec: "ZSTD", level: 3 },
+            accessControl: payloadAccessControl,
+        });
+        const payloadBytesColumn = new AnalyticsTableColumn({
+            key: "payloadBytes",
+            title: "Payload Bytes",
+            description: "Compressed wire size of this chunk, summed by the finalizer into the header's metering signal",
+            required: true,
+            type: TableColumnType.UInt64,
+            codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        /*
+         * Per-chunk counters carried on the envelope, so the ingest worker can
+         * populate them WITHOUT decompressing the payload. The finalizer sums
+         * them into the header.
+         */
+        const counterColumns = [
+            "errorCount",
+            "rageClickCount",
+            "deadClickCount",
+            "errorClickCount",
+            "refreshRageCount",
+            "routeCount",
+        ].map((key) => {
+            return new AnalyticsTableColumn({
+                key: key,
+                title: key,
+                description: `Per-chunk ${key}, summed into the session header by the finalizer`,
+                required: true,
+                type: TableColumnType.Number,
+                codec: [{ codec: "T64" }, { codec: "ZSTD", level: 1 }],
+                accessControl: chunkAccessControl,
+            });
+        });
+        const retentionDateColumn = new AnalyticsTableColumn({
+            key: "retentionDate",
+            title: "Retention Date",
+            description: "TTL date, derived from the clamped session start so every chunk of a session expires on the same day and the partition can be dropped whole",
+            required: true,
+            type: TableColumnType.Date,
+            codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
+            accessControl: chunkAccessControl,
+        });
+        super({
+            tableName: AnalyticsTableName.RumSessionChunk,
+            tableEngine: AnalyticsTableEngine.ReplacingMergeTree,
+            singularName: "Session Replay Chunk",
+            pluralName: "Session Replay Chunks",
+            /*
+             * crudApiPath is deliberately omitted so the payload column is
+             * unreachable through generic CRUD, and enableMCP is deliberately
+             * unset so no auto-generated LLM tool can read recordings.
+             */
+            accessControl: {
+                read: CHUNK_METADATA_PERMISSIONS,
+                create: CHUNK_CREATE_PERMISSIONS,
+                update: [],
+                delete: [
+                    Permission.ProjectOwner,
+                    Permission.ProjectAdmin,
+                    Permission.TelemetryAdmin,
+                    Permission.DeleteRumSessionReplay,
+                ],
+            },
+            tableBillingAccessControl: {
+                create: PlanType.Growth,
+                read: PlanType.Growth,
+                update: PlanType.Growth,
+                delete: PlanType.Growth,
+            },
+            tableColumns: [
+                projectIdColumn,
+                sessionIdColumn,
+                tabIdColumn,
+                chunkIndexColumn,
+                versionColumn,
+                rumApplicationIdColumn,
+                primaryEntityIdColumn,
+                primaryEntityTypeColumn,
+                sessionStartTimeColumn,
+                ...offsetColumns,
+                ...timeColumns,
+                eventCountColumn,
+                hasFullSnapshotColumn,
+                isFinalColumn,
+                isPinnedCopyColumn,
+                ...snapshotPartColumns,
+                ...descriptorColumns,
+                schemaVersionColumn,
+                payloadColumn,
+                payloadBytesColumn,
+                ...counterColumns,
+                retentionDateColumn,
+            ],
+            sortKeys: ["projectId", "sessionId", "tabId", "chunkIndex"],
+            primaryKeys: ["projectId", "sessionId", "tabId", "chunkIndex"],
+            /*
+             * Partitioned by EXPIRY, not by event time.
+             *
+             * ttl_only_drop_parts means a part is removed only once every row in
+             * it has expired. Partitioned by session date, a single day holding
+             * both a 7-day and a 90-day application survives 90 days — a ~4x
+             * disk multiplier on the fattest table in the system. Partitioned by
+             * expiry, each partition is "everything that dies on day X" and TTL
+             * drops it whole with zero mutations.
+             *
+             * Losing time-based pruning costs nothing here: the only read is by
+             * (session, chunk index), served by the sort key, and metering reads
+             * the header table instead. Retention is clamped to a closed set so
+             * this yields a handful of partitions per ingest day rather than one
+             * per distinct configured value.
+             */
+            partitionKey: "toYYYYMMDD(retentionDate)",
+            shardingKey: "cityHash64(projectId, sessionId)",
+            /*
+             * index_granularity = 128 rather than the default 8192.
+             *
+             * At the default, one granule of the payload column is ~10MB, so
+             * extracting a single 500KB session decompresses ~10MB — 20x read
+             * amplification on the hottest read in the feature. At 128 rows a
+             * granule is well under a megabyte, for a few MB of extra primary
+             * index per month.
+             */
+            tableSettings: "index_granularity = 128, ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
+            ttlExpression: "retentionDate DELETE",
+            defaultSortColumn: "chunkIndex",
+        });
+    }
+    get projectId() {
+        return this.getColumnValue("projectId");
+    }
+    set projectId(v) {
+        this.setColumnValue("projectId", v);
+    }
+    get sessionId() {
+        return this.getColumnValue("sessionId");
+    }
+    set sessionId(v) {
+        this.setColumnValue("sessionId", v);
+    }
+    get tabId() {
+        return this.getColumnValue("tabId");
+    }
+    set tabId(v) {
+        this.setColumnValue("tabId", v);
+    }
+    get chunkIndex() {
+        return this.getColumnValue("chunkIndex");
+    }
+    set chunkIndex(v) {
+        this.setColumnValue("chunkIndex", v);
+    }
+    get version() {
+        return this.getColumnValue("version");
+    }
+    set version(v) {
+        this.setColumnValue("version", v);
+    }
+    get rumApplicationId() {
+        return this.getColumnValue("rumApplicationId");
+    }
+    set rumApplicationId(v) {
+        this.setColumnValue("rumApplicationId", v);
+    }
+    get primaryEntityId() {
+        return this.getColumnValue("primaryEntityId");
+    }
+    set primaryEntityId(v) {
+        this.setColumnValue("primaryEntityId", v);
+    }
+    get primaryEntityType() {
+        return this.getColumnValue("primaryEntityType");
+    }
+    set primaryEntityType(v) {
+        this.setColumnValue("primaryEntityType", v);
+    }
+    get sessionStartTime() {
+        return this.getColumnValue("sessionStartTime");
+    }
+    set sessionStartTime(v) {
+        this.setColumnValue("sessionStartTime", v);
+    }
+    get chunkStartOffsetMs() {
+        return this.getColumnValue("chunkStartOffsetMs");
+    }
+    set chunkStartOffsetMs(v) {
+        this.setColumnValue("chunkStartOffsetMs", v);
+    }
+    get chunkEndOffsetMs() {
+        return this.getColumnValue("chunkEndOffsetMs");
+    }
+    set chunkEndOffsetMs(v) {
+        this.setColumnValue("chunkEndOffsetMs", v);
+    }
+    get chunkStartTime() {
+        return this.getColumnValue("chunkStartTime");
+    }
+    set chunkStartTime(v) {
+        this.setColumnValue("chunkStartTime", v);
+    }
+    get chunkEndTime() {
+        return this.getColumnValue("chunkEndTime");
+    }
+    set chunkEndTime(v) {
+        this.setColumnValue("chunkEndTime", v);
+    }
+    get eventCount() {
+        return this.getColumnValue("eventCount");
+    }
+    set eventCount(v) {
+        this.setColumnValue("eventCount", v);
+    }
+    get hasFullSnapshot() {
+        return this.getColumnValue("hasFullSnapshot");
+    }
+    set hasFullSnapshot(v) {
+        this.setColumnValue("hasFullSnapshot", v);
+    }
+    get isFinal() {
+        return this.getColumnValue("isFinal");
+    }
+    set isFinal(v) {
+        this.setColumnValue("isFinal", v);
+    }
+    get isPinnedCopy() {
+        return this.getColumnValue("isPinnedCopy");
+    }
+    set isPinnedCopy(v) {
+        this.setColumnValue("isPinnedCopy", v);
+    }
+    get snapshotPartIndex() {
+        return this.getColumnValue("snapshotPartIndex");
+    }
+    set snapshotPartIndex(v) {
+        this.setColumnValue("snapshotPartIndex", v);
+    }
+    get snapshotPartTotal() {
+        return this.getColumnValue("snapshotPartTotal");
+    }
+    set snapshotPartTotal(v) {
+        this.setColumnValue("snapshotPartTotal", v);
+    }
+    get recorderKind() {
+        return this.getColumnValue("recorderKind");
+    }
+    set recorderKind(v) {
+        this.setColumnValue("recorderKind", v);
+    }
+    get payloadEncoding() {
+        return this.getColumnValue("payloadEncoding");
+    }
+    set payloadEncoding(v) {
+        this.setColumnValue("payloadEncoding", v);
+    }
+    get schemaVersion() {
+        return this.getColumnValue("schemaVersion");
+    }
+    set schemaVersion(v) {
+        this.setColumnValue("schemaVersion", v);
+    }
+    get payload() {
+        return this.getColumnValue("payload");
+    }
+    set payload(v) {
+        this.setColumnValue("payload", v);
+    }
+    get payloadBytes() {
+        return this.getColumnValue("payloadBytes");
+    }
+    set payloadBytes(v) {
+        this.setColumnValue("payloadBytes", v);
+    }
+    get errorCount() {
+        return this.getColumnValue("errorCount");
+    }
+    set errorCount(v) {
+        this.setColumnValue("errorCount", v);
+    }
+    get rageClickCount() {
+        return this.getColumnValue("rageClickCount");
+    }
+    set rageClickCount(v) {
+        this.setColumnValue("rageClickCount", v);
+    }
+    get deadClickCount() {
+        return this.getColumnValue("deadClickCount");
+    }
+    set deadClickCount(v) {
+        this.setColumnValue("deadClickCount", v);
+    }
+    get errorClickCount() {
+        return this.getColumnValue("errorClickCount");
+    }
+    set errorClickCount(v) {
+        this.setColumnValue("errorClickCount", v);
+    }
+    get refreshRageCount() {
+        return this.getColumnValue("refreshRageCount");
+    }
+    set refreshRageCount(v) {
+        this.setColumnValue("refreshRageCount", v);
+    }
+    get routeCount() {
+        return this.getColumnValue("routeCount");
+    }
+    set routeCount(v) {
+        this.setColumnValue("routeCount", v);
+    }
+    get retentionDate() {
+        return this.getColumnValue("retentionDate");
+    }
+    set retentionDate(v) {
+        this.setColumnValue("retentionDate", v);
+    }
+};
+RumSessionChunk = __decorate([
+    OperationalResource(),
+    OwnedThrough("primaryEntityId", RumApplication, { includeProjectScope: true }),
+    __metadata("design:paramtypes", [])
+], RumSessionChunk);
+export default RumSessionChunk;
+//# sourceMappingURL=RumSessionChunk.js.map