@oneuptime/common 12.0.32 → 12.0.34

package/build/dist/Server/Utils/EmailRollup/EmailRollupFlushRunner.js

@@ -0,0 +1,733 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import DatabaseConfig from "../../DatabaseConfig";
+import Semaphore from "../../Infrastructure/Semaphore";
+import MailService from "../../Services/MailService";
+import ProjectService from "../../Services/ProjectService";
+import UserEmailService from "../../Services/UserEmailService";
+import UserNotificationEmailRollupBatchService from "../../Services/UserNotificationEmailRollupBatchService";
+import UserNotificationEmailRollupItemService from "../../Services/UserNotificationEmailRollupItemService";
+import UserNotificationSettingService from "../../Services/UserNotificationSettingService";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import PostgresErrorTranslator from "../Database/PostgresErrorTranslator";
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import { CLAIM_EPOCH_MINUTES, FLUSH_AFTER_MINUTES, MAX_BUCKETS_PER_TICK, MAX_DISCOVERY_PAGES_PER_TICK, MAX_ITEMS_PER_ROLLUP, MAX_ITEMS_SCANNED_PER_TICK, ROLLUP_JOB_NAME, ROLLUP_SEND_TIMEOUT_MS, ROLLUP_SWEEP_BUDGET_MS, ROLLUP_SWEEP_LOCK_NAMESPACE, ROLLUP_SWEEP_LOCK_TIMEOUT_MS, } from "./EmailRollupConstants";
+import { buildRollupEmail } from "./EmailRollupRenderer";
+import UserNotificationEmailRollupBatch, { RollupBatchStatus, } from "../../../Models/DatabaseModels/UserNotificationEmailRollupBatch";
+import URL from "../../../Types/API/URL";
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
+import ColumnLength from "../../../Types/Database/ColumnLength";
+import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../Types/Date";
+import EmailTemplateType from "../../../Types/Email/EmailTemplateType";
+import Text from "../../../Types/Text";
+/*
+ * What one flush attempt ended up doing. Returned rather than logged so the
+ * sweep can count outcomes and a test can assert on them without reading log
+ * lines.
+ */
+export var RollupFlushOutcome;
+(function (RollupFlushOutcome) {
+    // Another replica, or an earlier flush in this same epoch, owns the bucket.
+    RollupFlushOutcome["Collision"] = "collision";
+    // The claim won but nothing was still pending by the time it stamped.
+    RollupFlushOutcome["Empty"] = "empty";
+    // Deliberately not sent: the address or the project went away.
+    RollupFlushOutcome["Skipped"] = "skipped";
+    RollupFlushOutcome["Sent"] = "sent";
+    RollupFlushOutcome["Failed"] = "failed";
+})(RollupFlushOutcome || (RollupFlushOutcome = {}));
+const MILLISECONDS_PER_MINUTE = 60 * 1000;
+/*
+ * The batch table's statusMessage column is LongText, and a value longer than
+ * the column throws inside DatabaseService's own length check - which would
+ * turn "the send failed and we recorded why" into "the send failed and the
+ * recording of why also failed". Taken from the column length itself so the
+ * two can never drift apart.
+ */
+const STATUS_MESSAGE_MAX_LENGTH = ColumnLength.LongText;
+export default class EmailRollupFlushRunner {
+    /**
+     * The once-a-minute cron body: one sweep under a cross-replica lock.
+     *
+     * The lock is an OPTIMISATION, not a correctness mechanism. It exists so
+     * that N replicas do not all run the same scan every minute; the claim
+     * insert underneath it is what actually makes a rollup exactly-once. A held
+     * lock or an unreachable Redis therefore just skips the tick - the holder is
+     * covering the same work, and if nobody is, the next tick will.
+     */
+    static async runSweepUnderLock(options) {
+        let mutex = null;
+        try {
+            mutex = await Semaphore.lock({
+                key: ROLLUP_JOB_NAME,
+                namespace: ROLLUP_SWEEP_LOCK_NAMESPACE,
+                lockTimeout: ROLLUP_SWEEP_LOCK_TIMEOUT_MS,
+                acquireAttemptsLimit: 1,
+            });
+        }
+        catch (err) {
+            logger.debug(`${ROLLUP_JOB_NAME}: could not acquire the sweep lock; a sweep is already in flight (or Redis is unavailable). Skipping this run: ${err}`);
+            return null;
+        }
+        try {
+            return await EmailRollupFlushRunner.runSweep(options);
+        }
+        finally {
+            try {
+                await Semaphore.release(mutex);
+            }
+            catch (err) {
+                logger.error(`${ROLLUP_JOB_NAME}: error releasing the sweep lock: ${err}`);
+            }
+        }
+    }
+    /**
+     * One sweep: discover the due buckets oldest-first and flush each of them
+     * once. Never throws for one bucket's bad data - a bucket that fails is
+     * logged and the rest of the tick continues.
+     */
+    static async runSweep(options) {
+        const now = (options === null || options === void 0 ? void 0 : options.now) || OneUptimeDate.getCurrentDate();
+        const startMs = Date.now();
+        const cutoff = OneUptimeDate.addRemoveMinutes(now, FLUSH_AFTER_MINUTES * -1);
+        /*
+         * DISCOVERY IS BOUNDED, AND findBy IS THE POINT - not findAllBy.
+         * findAllBy pages to exhaustion, so a backlog spike would make this one
+         * query walk the entire pending table and blow the job's timeout part way
+         * through the loop, AFTER some buckets had already stamped their items but
+         * before they sent. An explicit limit means a backlog drains over several
+         * ticks instead, oldest first, which is also fairer across tenants.
+         *
+         * IT PAGES, THOUGH, AND THAT IS NOT AN OPTIMISATION. One page of
+         * MAX_ITEMS_SCANNED_PER_TICK rows sorted oldest-first can be entirely one
+         * saturated recipient: a single (project, user, address) with more than
+         * that many pending items would fill the page by itself, and every OTHER
+         * tenant's due rollup would be invisible for as long as the saturation
+         * lasted - a fleet-wide stall caused by one project, in exactly the storm
+         * this feature exists for. Paging past a hog costs a few extra indexed
+         * reads in a case that should never happen, and buys the guarantee that it
+         * cannot starve anybody.
+         */
+        const seen = new Set();
+        const buckets = [];
+        let itemsScanned = 0;
+        for (let page = 0; page < MAX_DISCOVERY_PAGES_PER_TICK; page++) {
+            const due = await UserNotificationEmailRollupItemService.findBy({
+                query: {
+                    sentAt: QueryHelper.isNull(),
+                    createdAt: QueryHelper.lessThanEqualTo(cutoff),
+                },
+                select: {
+                    projectId: true,
+                    userId: true,
+                    toEmail: true,
+                    createdAt: true,
+                },
+                sort: {
+                    createdAt: SortOrder.Ascending,
+                },
+                limit: MAX_ITEMS_SCANNED_PER_TICK,
+                skip: page * MAX_ITEMS_SCANNED_PER_TICK,
+                props: {
+                    isRoot: true,
+                },
+            });
+            itemsScanned += due.length;
+            /*
+             * Dedupe in JS, preserving first-seen order. The rows arrive
+             * oldest-first, so first-seen order IS oldest-bucket-first: the address
+             * that has been waiting longest is served first when the tick cannot
+             * serve everybody.
+             */
+            for (const item of due) {
+                if (!item.projectId || !item.userId || !item.toEmail) {
+                    continue;
+                }
+                const key = `${item.projectId.toString()}|${item.userId.toString()}|${item.toEmail.toString()}`;
+                if (seen.has(key)) {
+                    continue;
+                }
+                seen.add(key);
+                buckets.push({
+                    projectId: item.projectId,
+                    userId: item.userId,
+                    toEmail: item.toEmail,
+                });
+            }
+            // A short page is the end of the pending set; there is nothing behind it.
+            if (due.length < MAX_ITEMS_SCANNED_PER_TICK) {
+                break;
+            }
+            // Enough work for this tick. The rest keeps until the next one.
+            if (buckets.length >= MAX_BUCKETS_PER_TICK) {
+                break;
+            }
+        }
+        const bucketsThisTick = buckets.slice(0, MAX_BUCKETS_PER_TICK);
+        const stats = {
+            now: now,
+            cutoff: cutoff,
+            itemsScanned: itemsScanned,
+            bucketsDue: buckets.length,
+            bucketsProcessed: 0,
+            sent: 0,
+            empty: 0,
+            skipped: 0,
+            failed: 0,
+            claimCollisions: 0,
+            errors: 0,
+            budgetExhausted: false,
+        };
+        for (const bucket of bucketsThisTick) {
+            if (Date.now() - startMs > ROLLUP_SWEEP_BUDGET_MS) {
+                stats.budgetExhausted = true;
+                logger.warn(`${ROLLUP_JOB_NAME}: sweep budget exhausted after ${stats.bucketsProcessed} of ${bucketsThisTick.length} bucket(s); the rest are picked up next tick.`);
+                break;
+            }
+            try {
+                const outcome = await EmailRollupFlushRunner.flushBucket(bucket, now);
+                stats.bucketsProcessed = stats.bucketsProcessed + 1;
+                if (outcome === RollupFlushOutcome.Sent) {
+                    stats.sent = stats.sent + 1;
+                }
+                else if (outcome === RollupFlushOutcome.Empty) {
+                    stats.empty = stats.empty + 1;
+                }
+                else if (outcome === RollupFlushOutcome.Skipped) {
+                    stats.skipped = stats.skipped + 1;
+                }
+                else if (outcome === RollupFlushOutcome.Failed) {
+                    stats.failed = stats.failed + 1;
+                }
+                else {
+                    stats.claimCollisions = stats.claimCollisions + 1;
+                }
+            }
+            catch (err) {
+                /*
+                 * PER-BUCKET ISOLATION. One tenant's unreadable row, or a claim that
+                 * failed for a reason that is not a duplicate, must not cost every
+                 * other address on this tick its rollup.
+                 */
+                stats.errors = stats.errors + 1;
+                logger.error(err, {
+                    projectId: bucket.projectId.toString(),
+                    userId: bucket.userId.toString(),
+                });
+            }
+        }
+        return stats;
+    }
+    /**
+     * Claim one bucket's epoch, stamp its pending items, and send one rollup.
+     */
+    static async flushBucket(bucket, now) {
+        /*
+         * THE CLAIM KEY IS DERIVED FROM THE WALL CLOCK AND NOTHING ELSE.
+         *
+         * That is the whole reason the unique index is a real mutual exclusion:
+         * two replicas computing floor(now / epoch) at any instant inside the same
+         * epoch necessarily get the same value, so they necessarily collide.
+         * Derive it from the data instead - min(createdAt) of the pending rows,
+         * say - and two replicas reading slightly different snapshots would mint
+         * two different keys and both send.
+         *
+         * AND IT NEVER BLOCKS A LEGITIMATE FLUSH, because CLAIM_EPOCH_MINUTES
+         * equals FLUSH_AFTER_MINUTES. A bucket only becomes due once its OLDEST
+         * pending item is FLUSH_AFTER_MINUTES old, and a flush stamps every
+         * pending row in the bucket - so the next legitimate flush of the same
+         * bucket cannot happen until at least FLUSH_AFTER_MINUTES later, which is
+         * a strictly later epoch. What the index does block is a SECOND flush
+         * inside the same epoch, which is exactly the duplicate we want gone, and
+         * that is also where the hard "at most 60 / CLAIM_EPOCH_MINUTES rollups
+         * per hour per address" bound comes from.
+         */
+        const epochMs = CLAIM_EPOCH_MINUTES * MILLISECONDS_PER_MINUTE;
+        const epochStart = new Date(Math.floor(now.getTime() / epochMs) * epochMs);
+        const batch = new UserNotificationEmailRollupBatch();
+        batch.projectId = bucket.projectId;
+        batch.userId = bucket.userId;
+        batch.toEmail = bucket.toEmail;
+        batch.claimEpochStartsAt = epochStart;
+        batch.claimedAt = now;
+        batch.status = RollupBatchStatus.Claimed;
+        let created;
+        try {
+            created = await UserNotificationEmailRollupBatchService.create({
+                data: batch,
+                props: {
+                    isRoot: true,
+                },
+            });
+        }
+        catch (err) {
+            if (PostgresErrorTranslator.isUniqueViolation(err)) {
+                /*
+                 * Somebody else owns this bucket's epoch. Silently, and without
+                 * touching a single item row: the winner is about to stamp them.
+                 */
+                return RollupFlushOutcome.Collision;
+            }
+            throw err;
+        }
+        const batchId = created.id;
+        if (!batchId) {
+            throw new Error(`${ROLLUP_JOB_NAME}: the rollup batch claim returned no id.`);
+        }
+        /*
+         * EVERYTHING PAST THE CLAIM IS INSIDE THIS TRY, and that is the point of
+         * splitting the method here.
+         *
+         * Once the stamp lands, up to MAX_ITEMS_PER_ROLLUP notifications exist
+         * ONLY as an obligation to send this one email. If anything between the
+         * stamp and the send throws - the recipient re-validation, the project
+         * lookup, the dashboard URL, the renderer, a bug added here later - and
+         * the exception is merely logged by the sweep's per-bucket handler, those
+         * notifications are gone: stamped so nothing will ever pick them up again,
+         * with a batch row still saying Claimed and no trace of what happened.
+         * Recording Failed with the reason is the difference between a bounded,
+         * diagnosable loss and a silent one.
+         */
+        try {
+            return await EmailRollupFlushRunner.sendClaimedBatch(bucket, batchId, now);
+        }
+        catch (err) {
+            logger.error(err, {
+                projectId: bucket.projectId.toString(),
+                userId: bucket.userId.toString(),
+            });
+            try {
+                /*
+                 * Counted rather than assumed zero. If the throw happened after the
+                 * stamp, rows are already carrying this batch's id and the whole point
+                 * of recording Failed is to say how many notifications went with it; a
+                 * hard-coded 0 would make the ledger claim a loss of nothing. One
+                 * extra query, only on the failure path.
+                 */
+                const stampedCount = await UserNotificationEmailRollupItemService.countBy({
+                    query: {
+                        rollupBatchId: batchId,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
+                await EmailRollupFlushRunner.finish({
+                    batchId: batchId,
+                    status: RollupBatchStatus.Failed,
+                    itemCount: stampedCount.toNumber(),
+                    now: now,
+                    message: EmailRollupFlushRunner.describeError(err),
+                });
+            }
+            catch (finishErr) {
+                // Nothing further to try; the batch row keeps its Claimed status.
+                logger.error(finishErr);
+            }
+            return RollupFlushOutcome.Failed;
+        }
+    }
+    /*
+     * The half of a flush that runs after the epoch has been claimed: stamp the
+     * oldest pending rows into this batch, render them, and send. Separated so
+     * every failure in it is caught and recorded against the batch rather than
+     * escaping to the sweep loop, where it would leave stamped-but-unsent rows.
+     */
+    static async sendClaimedBatch(bucket, batchId, now) {
+        var _a;
+        /*
+         * STAMP BEFORE SEND, and bounded.
+         *
+         * The alternative - send first, stamp after - means a permanently broken
+         * SMTP server produces a rollup that grows every epoch and is re-sent
+         * every epoch, forever. That is precisely the storm this feature exists to
+         * stop, so it is not an acceptable failure mode here even though it is the
+         * more forgiving one elsewhere. The price is that a hard send failure
+         * loses that batch's notifications; that loss is bounded at
+         * MAX_ITEMS_PER_ROLLUP, is written down on the batch row with its reason,
+         * and the underlying resources are all still in the dashboard.
+         *
+         * The query deliberately DROPS the cutoff and takes every pending row for
+         * the bucket, including ones written seconds ago. Coalescing more is
+         * strictly better for the recipient, and it can only ever make an item's
+         * latency shorter than the schedule promised, never longer.
+         *
+         * THE OLDEST ROWS ARE SELECTED EXPLICITLY, AND THAT TAKES TWO STEPS.
+         * updateBy resolves the rows it will write with an internal _findBy that
+         * takes no sort from the caller (UpdateBy has no sort field) and therefore
+         * gets DatabaseService's default of createdAt DESCENDING. A bare
+         * `updateBy(..., limit: 500)` on an over-full bucket would consequently
+         * stamp the NEWEST five hundred and leave the oldest pending - a LIFO
+         * queue whose tail is never served, and whose tail is then hard-deleted
+         * unsent at ROLLUP_ITEM_RETENTION_DAYS. Selecting the ids oldest-first and
+         * updating exactly those makes the drain FIFO, which is the only ordering
+         * a notification queue may have.
+         */
+        const oldestPending = await UserNotificationEmailRollupItemService.findBy({
+            query: {
+                projectId: bucket.projectId,
+                userId: bucket.userId,
+                toEmail: bucket.toEmail,
+                sentAt: QueryHelper.isNull(),
+            },
+            select: {
+                _id: true,
+            },
+            sort: {
+                createdAt: SortOrder.Ascending,
+            },
+            limit: MAX_ITEMS_PER_ROLLUP,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const pendingIds = oldestPending
+            .map((item) => {
+            return item.id;
+        })
+            .filter((id) => {
+            return id !== null;
+        });
+        if (pendingIds.length === 0) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Empty,
+                itemCount: 0,
+                now: now,
+            });
+            return RollupFlushOutcome.Empty;
+        }
+        /*
+         * Read current preferences before consuming any queued items. A user may
+         * have turned an event's email off, or removed its setting, while this
+         * batch was waiting. Both mean no email, just as they do on the immediate
+         * send path. If this lookup fails, the items stay pending so a later epoch
+         * can retry instead of discarding them or mailing against unknown choices.
+         */
+        const settings = await UserNotificationSettingService.findBy({
+            query: {
+                projectId: bucket.projectId,
+                userId: bucket.userId,
+                alertByEmail: true,
+            },
+            select: {
+                eventType: true,
+            },
+            limit: LIMIT_PER_PROJECT,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const enabledEventTypes = new Set();
+        for (const setting of settings) {
+            if (setting.eventType) {
+                enabledEventTypes.add(setting.eventType);
+            }
+        }
+        const stamped = await UserNotificationEmailRollupItemService.updateBy({
+            query: {
+                _id: QueryHelper.any(pendingIds.map((id) => {
+                    return id.toString();
+                })),
+                /*
+                 * Belt and braces with the id list: a row that somebody else stamped
+                 * between the select above and this write must not be re-stamped
+                 * into this batch, or it would appear in two rollups.
+                 */
+                sentAt: QueryHelper.isNull(),
+            },
+            data: {
+                sentAt: now,
+                rollupBatchId: batchId,
+            },
+            limit: MAX_ITEMS_PER_ROLLUP,
+            skip: 0,
+            props: {
+                isRoot: true,
+                ignoreHooks: true,
+            },
+        });
+        if (stamped === 0) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Empty,
+                itemCount: 0,
+                now: now,
+            });
+            return RollupFlushOutcome.Empty;
+        }
+        /*
+         * READ BACK BY BATCH, never by "still pending for this bucket": this is
+         * what guarantees the email describes exactly the rows this attempt owns.
+         */
+        const claimedItems = await UserNotificationEmailRollupItemService.findBy({
+            query: {
+                rollupBatchId: batchId,
+            },
+            select: {
+                eventType: true,
+                rollupCategory: true,
+                subject: true,
+                viewLink: true,
+                createdAt: true,
+            },
+            sort: {
+                createdAt: SortOrder.Ascending,
+            },
+            limit: MAX_ITEMS_PER_ROLLUP,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        if (claimedItems.length === 0) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Empty,
+                itemCount: 0,
+                now: now,
+            });
+            return RollupFlushOutcome.Empty;
+        }
+        const items = claimedItems.filter((item) => {
+            return Boolean(item.eventType && enabledEventTypes.has(item.eventType));
+        });
+        if (items.length === 0) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Skipped,
+                itemCount: 0,
+                now: now,
+                message: "email notifications are no longer enabled for these events",
+            });
+            return RollupFlushOutcome.Skipped;
+        }
+        /*
+         * RE-VALIDATE THE RECIPIENT. The address was snapshotted minutes ago when
+         * the first item was queued, and the individual sends this rollup replaces
+         * would each have re-read the verified set at send time. An address that
+         * has since been unverified or removed must not be handed a list of this
+         * project's incident subjects.
+         */
+        const verified = await UserEmailService.findBy({
+            query: {
+                userId: bucket.userId,
+                projectId: bucket.projectId,
+                isVerified: true,
+            },
+            select: {
+                email: true,
+            },
+            limit: LIMIT_PER_PROJECT,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const stillVerified = verified.some((userEmail) => {
+            var _a;
+            return ((_a = userEmail.email) === null || _a === void 0 ? void 0 : _a.toString()) === bucket.toEmail.toString();
+        });
+        if (!stillVerified) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Skipped,
+                itemCount: items.length,
+                now: now,
+                message: "address is no longer a verified email for this user",
+            });
+            return RollupFlushOutcome.Skipped;
+        }
+        /*
+         * The project name is READ, not snapshotted at enqueue: one query per
+         * rollup email, always showing the project's current name, and it doubles
+         * as the "has this project been deleted while items were queued?" check.
+         */
+        const project = await ProjectService.findOneById({
+            id: bucket.projectId,
+            select: {
+                name: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        if (!project) {
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Skipped,
+                itemCount: items.length,
+                now: now,
+                message: "project not found",
+            });
+            return RollupFlushOutcome.Skipped;
+        }
+        const dashboardUrl = await DatabaseConfig.getDashboardUrl();
+        /*
+         * addRoute mutates the URL it is called on, so each link starts from a
+         * fresh URL parsed out of the dashboard URL's string rather than from the
+         * shared instance.
+         */
+        const projectHomeLink = URL.fromString(dashboardUrl.toString())
+            .addRoute(`/${bucket.projectId.toString()}/home`)
+            .toString();
+        const preferencesLink = URL.fromString(dashboardUrl.toString())
+            .addRoute(`/${bucket.projectId.toString()}/user-settings/notification-settings`)
+            .toString();
+        const built = buildRollupEmail({
+            projectName: (_a = project.name) !== null && _a !== void 0 ? _a : "",
+            projectHomeLink: projectHomeLink,
+            preferencesLink: preferencesLink,
+            items: items,
+        });
+        try {
+            /*
+             * AWAITED, unlike the fire-and-forget individual sends: a rollup that
+             * failed has to be recorded as Failed, and the only way to know is to
+             * wait for it.
+             *
+             * The options carry projectId and userId and NOTHING else. projectId is
+             * required for MailService to write an EmailLog row at all, so omitting
+             * it would make rollups invisible in Notification Logs; and a rollup
+             * spans many resources, so there is no single incident, alert or monitor
+             * it could honestly be correlated with.
+             */
+            await EmailRollupFlushRunner.withSendTimeout(MailService.sendMail({
+                toEmail: bucket.toEmail,
+                subject: built.subject,
+                templateType: EmailTemplateType.NotificationRollup,
+                vars: built.vars,
+            }, {
+                projectId: bucket.projectId,
+                userId: bucket.userId,
+            }));
+        }
+        catch (err) {
+            logger.error(err, {
+                projectId: bucket.projectId.toString(),
+                userId: bucket.userId.toString(),
+            });
+            await EmailRollupFlushRunner.finish({
+                batchId: batchId,
+                status: RollupBatchStatus.Failed,
+                itemCount: items.length,
+                now: now,
+                message: EmailRollupFlushRunner.describeError(err),
+            });
+            return RollupFlushOutcome.Failed;
+        }
+        await EmailRollupFlushRunner.finish({
+            batchId: batchId,
+            status: RollupBatchStatus.Sent,
+            itemCount: items.length,
+            now: now,
+        });
+        return RollupFlushOutcome.Sent;
+    }
+    /**
+     * Close out a claim row. sentAt is stamped only for a real send, so a batch
+     * row with a NULL sentAt and a non-Claimed status is always a rollup that
+     * did not go out, and statusMessage says why.
+     */
+    static async finish(data) {
+        var _a;
+        const update = {
+            status: data.status,
+            itemCount: data.itemCount,
+        };
+        if (data.status === RollupBatchStatus.Sent) {
+            update.sentAt = data.now;
+        }
+        if (data.message !== undefined) {
+            /*
+             * Guarded assignment: exactOptionalPropertyTypes means an optional
+             * property will not take an explicit undefined.
+             */
+            update.statusMessage =
+                (_a = Text.truncate(data.message, STATUS_MESSAGE_MAX_LENGTH)) !== null && _a !== void 0 ? _a : "";
+        }
+        await UserNotificationEmailRollupBatchService.updateOneById({
+            id: data.batchId,
+            data: update,
+            props: {
+                isRoot: true,
+            },
+        });
+    }
+    /*
+     * A thrown value is `unknown`, and the one thing the batch row must never do
+     * is fail to record a failure - so this never touches anything that could
+     * throw on a hostile value.
+     */
+    static describeError(err) {
+        if (err instanceof Error && err.message) {
+            return err.message;
+        }
+        if (typeof err === "string") {
+            return err;
+        }
+        return "unknown error";
+    }
+    /*
+     * Rejects if the send has not settled within ROLLUP_SEND_TIMEOUT_MS. See
+     * that constant for why a hung send is worse here than anywhere else.
+     *
+     * The underlying request is NOT cancelled - nothing in the stack can cancel
+     * it - so the mail may still go out afterwards. That is the right trade: the
+     * cost of being wrong is one duplicate rollup for one recipient, and the
+     * cost of not doing it is every rollup on the install stopping forever. The
+     * batch is recorded Failed either way, so the duplicate is explicable.
+     *
+     * The timer is always cleared, including on the happy path, so a pending
+     * setTimeout can never hold the worker process open at shutdown.
+     */
+    static async withSendTimeout(send) {
+        let timer = undefined;
+        const timeout = new Promise((_resolve, reject) => {
+            timer = setTimeout(() => {
+                reject(new Error(`${ROLLUP_JOB_NAME}: the rollup send did not complete within ${ROLLUP_SEND_TIMEOUT_MS}ms.`));
+            }, ROLLUP_SEND_TIMEOUT_MS);
+        });
+        try {
+            await Promise.race([send, timeout]);
+        }
+        finally {
+            if (timer !== undefined) {
+                clearTimeout(timer);
+            }
+            /*
+             * Promise.race leaves the loser unhandled. When the timeout wins, the
+             * send is still in flight and will eventually settle; without this its
+             * rejection would surface as an unhandled rejection and, depending on
+             * the runtime's setting, take the worker down.
+             */
+            send.catch((err) => {
+                logger.error(err);
+            });
+        }
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], EmailRollupFlushRunner, "runSweepUnderLock", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], EmailRollupFlushRunner, "runSweep", null);
+//# sourceMappingURL=EmailRollupFlushRunner.js.map