@substrat-run/kernel 0.159.0 → 0.161.0

package/dist/platform-sweep.js

@@ -1,6 +1,9 @@
 import { drainedEvent, instant, substratError } from '@substrat-run/contracts';
 import { STORAGE_GAUGE_PRUNE_BATCH, STORED_SCOPE_STATUSES, holdsStore } from './storage-gauge.js';
-import { assertRowLimit, backoffAt, TELEMETRY_PRUNE_BATCH } from './scope-host.js';
+import { BILLING_ROLLUP_RETENTION_MONTHS, billingRollupHorizon, utcDayOf } from './billing-retention.js';
+import { METER_DAYS_PRUNE_BATCH, METER_SNAPSHOT_TIMEOUT_MS } from './meter-days.js';
+import { REQUEST_ROLLUP_DAYS_PER_PASS, REQUEST_ROLLUP_LOOKBACK_DAYS, REQUEST_ROLLUP_MAX_LOOKBACK_DAYS, REQUEST_ROLLUP_PRUNE_BATCH, REQUEST_ROLLUP_READ_TIMEOUT_MS, REQUEST_ROLLUP_RETRY_MS, REQUEST_ROLLUP_SETTLE_MS, isRequestRollupSaturated, requestRollupDueDays, } from './request-rollup.js';
+import { assertRowLimit, backoffAt, fetchLikeOf, TELEMETRY_PRUNE_BATCH } from './scope-host.js';
 import { MIGRATION_FLAG_THRESHOLD, migrationFleet, migrationProgress, scopeMigrationState } from './migration-progress.js';
 import { UNDRAINED_SKIPPED_IDS } from './outbox-event.js';
 import { resolveVerticalInstanceFrom } from './peer.js';
@@ -68,6 +71,15 @@ function reconcileWindow(behind, batch, rng) {
 export class StorageReadUnsupported extends Error {
     unsupported = true;
 }
+/**
+ * What a delegated event read throws when the scope's deployment says it does not hold the scope
+ * (#2544). Standing, like `StorageReadUnsupported`: the drain counts it in `EventDrainReport.notHeld`
+ * and keeps it out of `report.errors`. Recognised by its flag, not `instanceof`, for the same reason.
+ */
+export class ScopeNotHeld extends Error {
+    notHeld = true;
+}
+const isNotHeld = (err) => typeof err === 'object' && err !== null && err.notHeld === true;
 const isUnsupported = (err) => typeof err === 'object' && err !== null && err.unsupported === true;
 /** `work`, or a rejection once `ms` have passed. The work is not cancelled; the pass stops waiting. */
 async function withinDeadline(work, ms, what) {
@@ -188,6 +200,36 @@ function assertSweepBatches(options) {
     if (gauge?.batch !== undefined && !(Number.isInteger(gauge.batch) && gauge.batch >= 0)) {
         throw substratError('validation_failed', `storageGauge.batch must be a non-negative integer, got ${String(gauge.batch)}`);
     }
+    const rollup = options.requestRollup;
+    if (rollup?.pruneBatch !== undefined)
+        assertRowLimit('requestRollup.pruneBatch', rollup.pruneBatch);
+    if (rollup?.batch !== undefined && !(Number.isInteger(rollup.batch) && rollup.batch >= 0)) {
+        throw substratError('validation_failed', `requestRollup.batch must be a non-negative integer, got ${String(rollup.batch)}`);
+    }
+    if (rollup?.retentionMonths !== undefined)
+        assertBillingRetention('requestRollup.retentionMonths', rollup.retentionMonths);
+    // Each of these corrupts the kept history silently rather than failing: a lookback past what
+    // Analytics Engine keeps records aged-out days as days without traffic, and a negative settle
+    // reads the open day and freezes it. So they are refused here, before anything is read.
+    const lookback = rollup?.lookbackDays;
+    if (lookback !== undefined && !(Number.isInteger(lookback) && lookback >= 1 && lookback <= REQUEST_ROLLUP_MAX_LOOKBACK_DAYS)) {
+        throw substratError('validation_failed', `requestRollup.lookbackDays must be a whole number of days from 1 to ${REQUEST_ROLLUP_MAX_LOOKBACK_DAYS}, got ${String(lookback)}`);
+    }
+    for (const name of ['settleMs', 'retryMs']) {
+        const ms = rollup?.[name];
+        if (ms !== undefined && !(Number.isFinite(ms) && ms >= 0)) {
+            throw substratError('validation_failed', `requestRollup.${name} must be a non-negative number of ms, got ${String(ms)}`);
+        }
+    }
+    if (rollup?.readTimeoutMs !== undefined)
+        assertPositiveMs('requestRollup.readTimeoutMs', rollup.readTimeoutMs);
+    const snapshot = options.meterSnapshot;
+    if (snapshot?.pruneBatch !== undefined)
+        assertRowLimit('meterSnapshot.pruneBatch', snapshot.pruneBatch);
+    if (snapshot?.retentionMonths !== undefined)
+        assertBillingRetention('meterSnapshot.retentionMonths', snapshot.retentionMonths);
+    if (snapshot?.timeoutMs !== undefined)
+        assertPositiveMs('meterSnapshot.timeoutMs', snapshot.timeoutMs);
     const orphans = options.orphanAttachments;
     if (orphans?.batch !== undefined && !(Number.isInteger(orphans.batch) && orphans.batch >= 0)) {
         throw substratError('validation_failed', `orphanAttachments.batch must be a non-negative integer, got ${String(orphans.batch)}`);
@@ -196,6 +238,69 @@ function assertSweepBatches(options) {
         throw substratError('validation_failed', `orphanAttachments.mode must be 'dry-run' or 'delete', got ${String(orphans.mode)}`);
     }
 }
+/** A billing retention option, refused unless `billingRollupHorizon` takes it. */
+function assertBillingRetention(name, months) {
+    try {
+        billingRollupHorizon(0, months);
+    }
+    catch (err) {
+        throw substratError('validation_failed', `${name}: ${message(err)}`);
+    }
+}
+/** A deadline option, refused unless it is a positive, finite number of ms. */
+function assertPositiveMs(name, ms) {
+    if (!(Number.isFinite(ms) && ms > 0)) {
+        throw substratError('validation_failed', `${name} must be a positive number of ms, got ${String(ms)}`);
+    }
+}
+/**
+ * How many unreadable directory rows one pass names before it only says there are more: the
+ * sweep's `errors`, and the hostname reconcile's `unreadableScopes`, so each pass's report stays
+ * bounded however many rows a directory holds.
+ */
+export const UNREADABLE_SCOPE_ROWS_NAMED = 20;
+/**
+ * The scope read every fleet-wide phase of one pass shares. One row that does not parse used to
+ * throw out of `listScopes` for the whole list, and since no phase reads the directory inside a
+ * try, out of the pass: every phase after it went unrun until the row was fixed by hand. Read
+ * through `listScopesReadable`, the row is stepped over, and named in `errors` ONCE per pass
+ * however many phases met it, by its id and the fields that failed, never their values. At most
+ * `UNREADABLE_SCOPE_ROWS_NAMED` are named, then one line says there are more. A host that
+ * predates `listScopesReadable` is read through `listScopes`, strict as before.
+ */
+function scopeReaderFor(admin, actor, report) {
+    const readable = admin.listScopesReadable?.bind(admin);
+    if (!readable)
+        return (filter) => admin.listScopes(actor, filter);
+    const named = new Set();
+    let capped = false;
+    return async (filter) => {
+        const { scopes, unreadable } = await readable(actor, filter);
+        for (const row of unreadable) {
+            if (capped)
+                break;
+            if (named.has(row.scopeId))
+                continue;
+            if (named.size < UNREADABLE_SCOPE_ROWS_NAMED) {
+                named.add(row.scopeId);
+                report.errors.push({
+                    kind: 'scope-row',
+                    id: row.scopeId,
+                    error: `the directory row of tenant ${row.tenantId} does not read as a scope (${row.fields.join(', ')}); stepped over this pass`,
+                });
+            }
+            else {
+                capped = true;
+                report.errors.push({
+                    kind: 'scope-row',
+                    id: '*',
+                    error: `more than ${UNREADABLE_SCOPE_ROWS_NAMED} directory rows do not read as a scope; only the first ${UNREADABLE_SCOPE_ROWS_NAMED} are named this pass`,
+                });
+            }
+        }
+        return scopes;
+    };
+}
 export async function runPlatformSweep(host, options) {
     assertSweepBatches(options);
     const concurrency = options.concurrency ?? 8;
@@ -219,8 +324,13 @@ export async function runPlatformSweep(host, options) {
         findings: null,
         storage: null,
         orphanAttachments: null,
+        requestRollup: null,
+        meterSnapshot: null,
         errors: [],
     };
+    // Every fleet-wide read of the directory below goes through this, so one unreadable row is
+    // stepped over by each phase and named once. The tenant reap does not: see there.
+    const listScopes = scopeReaderFor(host.admin, options.actor, report);
     // -- migration reconciliation (kernel-design §5.3, #49) ---------------------
     // FIRST, deliberately: `drainDue` wakes scopes (and wake migrates lazily), so
     // running this phase after it would mean every failed scope gets its attempt
@@ -244,9 +354,7 @@ export async function runPlatformSweep(host, options) {
         const now = new Date().toISOString();
         // `provisioning` included: a scope stuck there because its migration failed
         // is precisely a straggler (the pure adapter's getScope makes the same call).
-        const listed = await host.admin.listScopes(options.actor, {
-            status: ['active', 'provisioning'],
-        });
+        const listed = await listScopes({ status: ['active', 'provisioning'] });
         const fleet = migrationFleet(listed);
         // Due = behind and never-failed (wake it — that IS the sweep's job), or
         // failed with its backoff window elapsed. A failed scope inside its window
@@ -328,7 +436,7 @@ export async function runPlatformSweep(host, options) {
     const reachedThisPass = new Map();
     const viaPlatformDrain = Boolean(options.drainPlatformRequestsFn);
     if (options.drainRetries !== false) {
-        const scopes = await host.admin.listScopes(options.actor, { status: 'active' });
+        const scopes = await listScopes({ status: 'active' });
         await mapBounded(scopes, concurrency, async (s) => {
             if (failedThisPass.has(s.id))
                 return; // fails closed — draining is only noise
@@ -356,7 +464,7 @@ export async function runPlatformSweep(host, options) {
     const drainedThisPass = new Map();
     const drain = options.drainPlatformRequestsFn;
     if (drain) {
-        const scopes = await host.admin.listScopes(options.actor, { status: 'active' });
+        const scopes = await listScopes({ status: 'active' });
         await mapBounded(scopes, concurrency, async (s) => {
             if (failedThisPass.has(s.id)) {
                 report.platformRequestTotals.skipped += 1;
@@ -381,7 +489,7 @@ export async function runPlatformSweep(host, options) {
     if (options.storageGauge &&
         typeof host.admin.recordScopeStorage === 'function' &&
         typeof host.admin.listScopeStorageAttempts === 'function') {
-        report.storage = await sweepStorageGauge(host.admin, options, options.storageGauge, reachedThisPass, report);
+        report.storage = await sweepStorageGauge(host.admin, options, options.storageGauge, reachedThisPass, report, listScopes);
     }
     // -- provision reconcile (#1172, #1653) -------------------------------------
     // A vertical's `onProvision` runs once per scope, at install. So a scope serving code
@@ -404,7 +512,7 @@ export async function runPlatformSweep(host, options) {
             unsupported: { count: 0, scopeIds: [] },
             deferred: 0,
         };
-        const scopes = (await host.admin.listScopes(options.actor, { status: 'active' })).filter((s) => isPrimaryScope(s) && !failedThisPass.has(s.id));
+        const scopes = (await listScopes({ status: 'active' })).filter((s) => isPrimaryScope(s) && !failedThisPass.has(s.id));
         const serving = await servingPointersFor(host.admin, options.actor, scopes);
         const behind = [];
         for (const s of scopes) {
@@ -484,21 +592,27 @@ export async function runPlatformSweep(host, options) {
             // Primaries only (`isPrimaryScope`, #2005) — a fork, a snapshot or a preview of either
             // kind is a copy or a throwaway; firing its schedules would run real recurring side
             // effects off it. Lineage alone (`forkedFrom === null`) let a clean-room preview through.
-            const scopes = (await host.admin.listScopes(options.actor, { status: 'active' })).filter(isPrimaryScope);
+            const scopes = (await listScopes({ status: 'active' })).filter(isPrimaryScope);
             await mapBounded(scopes, concurrency, async (s) => {
                 if (failedThisPass.has(s.id))
                     return;
                 let touched = false;
                 let platformRequests = 0;
+                // #2517: the requests this scope's purge runs enqueued so far in the tick, across modules —
+                // nothing drains between them, so a later purge head the cap refuses waits for the next tick.
+                let purgeRequests = 0;
                 for (const reg of registrations) {
                     try {
-                        const r = await host.runDueSchedules(reg.moduleId, s.tenantId, s.id);
+                        const r = await host.runDueSchedules(reg.moduleId, s.tenantId, s.id, { earlierPurgeRequests: purgeRequests });
                         if (r.fired > 0 || r.failed > 0)
                             touched = true;
                         schedules.fired += r.fired;
                         schedules.skipped += r.skipped;
                         schedules.failed += r.failed;
+                        if (r.deferred)
+                            schedules.deferred = (schedules.deferred ?? 0) + r.deferred;
                         platformRequests += r.platformRequests ?? 0;
+                        purgeRequests += r.purgeRequests ?? 0;
                         for (const e of r.errors) {
                             report.errors.push({ kind: 'schedule', id: `${s.id}:${e.operation}`, error: e.error });
                         }
@@ -566,7 +680,7 @@ export async function runPlatformSweep(host, options) {
         const freshRegs = host.registeredFreshness().filter((r) => r.freshness.length > 0);
         if (freshRegs.length > 0) {
             // The same primaries the schedule phase fires on (#2005).
-            const scopes = (await host.admin.listScopes(options.actor, { status: 'active' })).filter(isPrimaryScope);
+            const scopes = (await listScopes({ status: 'active' })).filter(isPrimaryScope);
             await mapBounded(scopes, concurrency, async (s) => {
                 if (failedThisPass.has(s.id))
                     return;
@@ -600,7 +714,7 @@ export async function runPlatformSweep(host, options) {
     // After the migration phase and skipping what failed there, like every phase that reaches a
     // vertical. Opt-in, and its own window per pass: it lists a bucket prefix and wakes a scope.
     if (options.orphanAttachments) {
-        const scopes = (await host.admin.listScopes(options.actor, { status: 'active' })).filter((s) => isPrimaryScope(s) && !failedThisPass.has(s.id));
+        const scopes = (await listScopes({ status: 'active' })).filter((s) => isPrimaryScope(s) && !failedThisPass.has(s.id));
         report.orphanAttachments = await sweepOrphanAttachmentWindow(host, options, options.orphanAttachments, scopes, report);
     }
     if (options.gcSnapshots !== false) {
@@ -612,7 +726,7 @@ export async function runPlatformSweep(host, options) {
         // `deleteSnapshot` re-checks this same predicate, so a mislabeled row fails closed there,
         // never silently deletes.
         const now = new Date().toISOString();
-        const scopes = await host.admin.listScopes(options.actor);
+        const scopes = await listScopes();
         const expired = scopes.filter((s) => (s.forkedFrom !== null || s.kind === 'preview') && s.expiresAt !== null && s.expiresAt <= now);
         const reap = options.deleteSnapshotFn ??
             ((tenantId, scopeId) => host.deleteSnapshot(options.actor, tenantId, scopeId));
@@ -635,7 +749,7 @@ export async function runPlatformSweep(host, options) {
     // status below the seam, so a row that changed underneath fails closed there.
     if (options.reapArchivedAfterDays !== undefined && options.reapArchivedAfterDays >= 0) {
         const cutoff = new Date(Date.now() - options.reapArchivedAfterDays * 86_400_000).toISOString();
-        const archived = await host.admin.listScopes(options.actor, { status: ['archived'] });
+        const archived = await listScopes({ status: ['archived'] });
         const due = archived.filter((s) => s.archivedAt !== null && s.archivedAt <= cutoff);
         const reap = options.reapScopeFn ??
             // Retention reap is a deliberate policy on already-archived scopes: force past the
@@ -672,6 +786,11 @@ export async function runPlatformSweep(host, options) {
                 // Co-located default: reap every one of the tenant's scopes through the scope
                 // seam (archive-if-needed first — reapScope only accepts `archived`), then clear
                 // the tenant's directory rows. A scope already `reaped` is skipped.
+                //
+                // STRICT, unlike every other read in this pass: `reapTenant` below clears the tenant's
+                // directory, so a scope this loop stepped over would never have its storage reaped and
+                // would lose the rows that find it. A row that does not parse fails this tenant's reap
+                // instead, into `errors` as `reap-tenant`, and the tenant waits for the row to be fixed.
                 const scopes = await host.admin.listScopes(options.actor, { tenantId });
                 for (const s of scopes) {
                     if (s.status === 'reaped')
@@ -694,7 +813,7 @@ export async function runPlatformSweep(host, options) {
         });
     }
     const connections = await host.admin.listConnections(options.actor, {});
-    const landings = options.connectionLandings ? new LandingPass(host, options.actor, options.connectionLandings, concurrency) : null;
+    const landings = options.connectionLandings ? new LandingPass(host, options.actor, options.connectionLandings, concurrency, listScopes) : null;
     await mapBounded(connections, concurrency, async (c) => {
         // #1232: the connection's durable sweep record. A connection row carries no
         // scope (a connection spans scopes by construction) and no version.
@@ -734,7 +853,8 @@ export async function runPlatformSweep(host, options) {
             report.connectionLandings.failed += landed.failed.length;
         }
         try {
-            const result = await sweeper(host, c.id, { fetch: options.fetch });
+            // #2498: a sized stream body the connector sends reaches the injected fetch converted.
+            const result = await sweeper(host, c.id, { fetch: fetchLikeOf(options.fetch) });
             const answer = options.connectionLandings?.binders[c.provider] ? connectorSweepAnswerOf(result) : null;
             report.connectionsSwept += 1;
             // #2316: a resolved sweep is not a landed one. A scope the connector could not land in, or a
@@ -769,7 +889,7 @@ export async function runPlatformSweep(host, options) {
     // own outbox. An import is never re-emitted into the consumer's outbox, so the two phases
     // never ship the same event twice.
     if (options.crossVertical) {
-        report.crossVertical = await sweepCrossVertical(host, options, options.crossVertical, failedThisPass, report);
+        report.crossVertical = await sweepCrossVertical(host, options, options.crossVertical, failedThisPass, report, listScopes);
     }
     // -- drain each scope's domain events to Tier 2 (#1334, master-plan §5.3) ---
     // Before the access-log drain below, which is deliberately last: this phase
@@ -785,7 +905,7 @@ export async function runPlatformSweep(host, options) {
         // fleet-wide fault.
         const configured = options.eventDrainBatch ?? EVENT_DRAIN_BATCH;
         const budget = Number.isFinite(configured) && configured >= 1 ? Math.floor(configured) : EVENT_DRAIN_BATCH;
-        const drainable = await host.admin.listScopes(options.actor, { status: 'active' });
+        const drainable = await listScopes({ status: 'active' });
         await mapBounded(drainable, concurrency, async (s) => {
             try {
                 const { shipped, skipped } = await drainScopeEvents(host, options, sink, {
@@ -806,6 +926,10 @@ export async function runPlatformSweep(host, options) {
                     report.eventDrain.incomplete += 1;
             }
             catch (err) {
+                if (isNotHeld(err)) {
+                    (report.eventDrain.notHeld ??= []).push({ tenantId: s.tenantId, scopeId: s.id });
+                    return;
+                }
                 // One scope's failure never sinks the sweep, and never stamps: a throw
                 // anywhere in read→ship→mark leaves its events exactly where they were,
                 // to be taken again next tick.
@@ -836,6 +960,26 @@ export async function runPlatformSweep(host, options) {
             report.errors.push({ kind: 'findings', id: 'findings', error: message(err) });
         }
     }
+    // -- the daily meter snapshot (#2501) -----------------------------------------
+    // After all the scheduled work, so the day's reading is the directory as this pass leaves it,
+    // reaps included, and just before the request rollup: the snapshot is one directory transaction,
+    // and a slow Analytics Engine answer must not push its reading later in the day. Its one read of
+    // the day records is shipped by the access-log drain below, in the same pass.
+    if (options.meterSnapshot &&
+        typeof host.admin.recordMeterDay === 'function' &&
+        typeof host.admin.listMeterDayTakes === 'function') {
+        report.meterSnapshot = await sweepMeterSnapshot(host.admin, options.actor, options.meterSnapshot, report);
+    }
+    // -- the daily request rollup (#2501) ----------------------------------------
+    // After all the scheduled work, deliberately: it waits on Analytics Engine, an outside service,
+    // and nothing in the pass depends on it, so a slow answer must not hold back the work that has
+    // to happen every pass. Each read is bounded (`RequestRollupSweepOptions.readTimeoutMs`), so the
+    // most it can delay is the access-log drain below, which then ships this phase's own reads too.
+    if (options.requestRollup &&
+        typeof host.admin.recordRequestDay === 'function' &&
+        typeof host.admin.listRequestRollupDays === 'function') {
+        report.requestRollup = await sweepRequestRollup(host.admin, options.actor, options.requestRollup, report);
+    }
     // -- drain the staff access log to Tier 2, then prune it (K-24, §4.4) --------
     // LAST, deliberately: every phase above reads the directory through the audited
     // seam, so each one writes access rows. Running the drain last means a pass ships
@@ -857,7 +1001,7 @@ export async function runPlatformSweep(host, options) {
  * One pass over every cross-vertical edge (#1705). Bounded like the event drain: one batch per
  * edge per pass, reported rather than looped, so one busy producer cannot starve the rest.
  */
-async function sweepCrossVertical(host, options, cv, failedThisPass, report, 
+async function sweepCrossVertical(host, options, cv, failedThisPass, report, listScopes, 
 /** Run only this producer's outgoing edges (the router kick). Absent: every edge. */
 only) {
     const out = {
@@ -889,7 +1033,7 @@ only) {
     //
     // A producer-scoped run reads only that tenant's scopes: both ends of an edge are in one
     // tenant, so nothing outside it can be a consumer of this producer or change how it resolves.
-    const scopes = (await host.admin.listScopes(options.actor, { status: 'active', ...(only ? { tenantId: only.tenantId } : {}) })).filter((s) => isPrimaryScope(s) && s.vertical !== null);
+    const scopes = (await listScopes({ status: 'active', ...(only ? { tenantId: only.tenantId } : {}) })).filter((s) => isPrimaryScope(s) && s.vertical !== null);
     // The kick names a scope, and the edges run are the ones whose producer RESOLVES to it. A
     // fork, a preview, a second install or a scope of another tenant is not a producer, so a
     // kick naming one runs nothing (the sweep would resolve the same way).
@@ -1044,7 +1188,8 @@ only) {
  */
 export async function runCrossVerticalFrom(host, options, producer) {
     const report = { errors: [] };
-    const crossVertical = await sweepCrossVertical(host, options, options.crossVertical, new Set(), report, producer);
+    const listScopes = scopeReaderFor(host.admin, options.actor, report);
+    const crossVertical = await sweepCrossVertical(host, options, options.crossVertical, new Set(), report, listScopes, producer);
     return { crossVertical, errors: report.errors };
 }
 /**
@@ -1590,7 +1735,7 @@ const cmp = (x, y) => (x < y ? -1 : x > y ? 1 : 0);
  * in one directory call, then prune past retention. A failed read is an error entry; its
  * scope keeps its last sample and waits a day like any other.
  */
-async function sweepStorageGauge(admin, options, gauge, reachedThisPass, report) {
+async function sweepStorageGauge(admin, options, gauge, reachedThisPass, report, listScopes) {
     const out = {
         reached: reachedThisPass.size,
         resting: 0,
@@ -1610,7 +1755,7 @@ async function sweepStorageGauge(admin, options, gauge, reachedThisPass, report)
         if (batch > 0) {
             // Not one archived before it ever held data (`holdsStore`): reading it would create the
             // spine it never had, and bill it.
-            const resting = (await admin.listScopes(options.actor, { status: STORED_SCOPE_STATUSES.filter((s) => s !== 'active') })).filter(holdsStore);
+            const resting = (await listScopes({ status: STORED_SCOPE_STATUSES.filter((s) => s !== 'active') })).filter(holdsStore);
             out.resting = resting.length;
             const candidates = [...reachedThisPass.values(), ...resting.filter((s) => !reachedThisPass.has(s.id))];
             const tried = new Map((await admin.listScopeStorageAttempts(options.actor)).map((a) => [a.scopeId, a.attemptedAt]));
@@ -1639,33 +1784,48 @@ async function sweepStorageGauge(admin, options, gauge, reachedThisPass, report)
                     }
                     if (!Number.isInteger(bytes) || bytes < 0)
                         throw new Error(`not a size: ${String(bytes)}`);
-                    // The store read is its own attempt. A failure here must not throw away the
-                    // database size just read, and must not replace the day's store rows.
-                    let stores;
-                    if (gauge.readStores) {
+                    // The store read and the attachment read are each their own: a failure of one must not
+                    // throw away the database size just read, nor the other's reading, nor replace what the
+                    // day already holds of it. A read the deployment cannot make is not a failure and not a
+                    // size of 0: what it would measure is left as it is. Neither waits for the other, so a
+                    // slow scope holds its slot for one deadline, not two.
+                    const beside = {};
+                    const timeoutMs = gauge.storeReadTimeoutMs ?? STORE_READ_TIMEOUT_MS;
+                    /** One read beside the database: its failure message, or nothing. */
+                    const readBeside = async (read, label, keep) => {
+                        if (!read)
+                            return undefined;
                         try {
-                            stores = await withinDeadline(gauge.readStores(s), gauge.storeReadTimeoutMs ?? STORE_READ_TIMEOUT_MS, `scope store read of ${s.id}`);
+                            const value = await withinDeadline(read.call(gauge, s), timeoutMs, `${label} of ${s.id}`);
+                            if (value !== null)
+                                keep(value);
                         }
                         catch (err) {
-                            if (isUnsupported(err)) {
-                                // Not a failure, and not a size: the database sample stands, the store rows stay.
-                                out.read += 1;
-                                out.unsupported += 1;
-                                attempts.push({ ...attempt, bytes });
-                                return;
-                            }
-                            out.failed += 1;
-                            attempts.push({ ...attempt, bytes, error: message(err) });
-                            report.errors.push({ kind: 'storage', id: s.id, error: message(err) });
-                            return;
+                            if (!isUnsupported(err))
+                                return message(err);
+                            out.unsupported += 1;
                         }
+                        return undefined;
+                    };
+                    // Listed in a fixed order, so a scope failing both always reports them the same way.
+                    const failures = (await Promise.all([
+                        readBeside(gauge.readStores, 'scope store read', (stores) => {
+                            beside.stores = stores;
+                        }),
+                        readBeside(gauge.readAttachments, 'attachment read', (attachments) => {
+                            if (!Number.isInteger(attachments) || attachments < 0)
+                                throw new Error(`not a size: ${String(attachments)}`);
+                            beside.attachments = attachments;
+                        }),
+                    ])).filter((failure) => failure !== undefined);
+                    if (failures.length > 0) {
+                        const error = failures.join('; ');
+                        out.failed += 1;
+                        attempts.push({ ...attempt, bytes, ...beside, error });
+                        report.errors.push({ kind: 'storage', id: s.id, error });
+                        return;
                     }
-                    if (stores === null) {
-                        attempts.push({ ...attempt, bytes });
-                    }
-                    else {
-                        attempts.push({ ...attempt, bytes, ...(stores ? { stores } : {}) });
-                    }
+                    attempts.push({ ...attempt, bytes, ...beside });
                     out.read += 1;
                 }
                 catch (err) {
@@ -1691,6 +1851,189 @@ async function sweepStorageGauge(admin, options, gauge, reachedThisPass, report)
     }
     return out;
 }
+const NO_ATTEMPT = { error: null, since: null, failures: 0 };
+/** The newest of a ledger's attempts (#2501), and the run of identical failures it belongs to when it failed. */
+function previousAttempt(attempts) {
+    let newest;
+    for (const a of attempts)
+        if (newest === undefined || a.attemptedAt > newest.attemptedAt)
+            newest = a;
+    return newest ? { error: newest.error, since: newest.failingSince ?? newest.attemptedAt, failures: newest.failingCount ?? 1 } : NO_ATTEMPT;
+}
+/**
+ * A failure's run (#2501): the previous attempt's run carried on when this failure repeats its error,
+ * a new run begun `at` otherwise. A repeated failure is a standing one, which reached the failure
+ * digest when its run began; the run keeps its start and length on the ledger and in the report,
+ * where the pass's log line shows them.
+ */
+function extendRun(previous, error, at) {
+    const repeated = error === previous.error;
+    return {
+        repeated,
+        run: { error, since: repeated && previous.since ? previous.since : at, failures: repeated ? previous.failures + 1 : 1 },
+    };
+}
+/**
+ * One pass of the request-rollup phase (#2501): read the oldest due days one at a time, record
+ * each day's counts or its failure, then prune past retention. A failed day writes nothing of the
+ * day, waits `retryMs` like any other attempt, and ends the pass's reads: the days after it stay
+ * due. A failure that repeats the previous attempt's is a standing one (a credential, a dataset
+ * name, an outage), reported when its streak began and not again until a read succeeds.
+ */
+async function sweepRequestRollup(admin, actor, rollup, report) {
+    const out = {
+        due: 0,
+        deferred: 0,
+        read: 0,
+        failed: 0,
+        repeated: 0,
+        streak: null,
+        saturated: 0,
+        saturatedDays: [],
+        recorded: 0,
+        pruned: 0,
+    };
+    const batch = rollup.batch ?? REQUEST_ROLLUP_DAYS_PER_PASS;
+    const months = rollup.retentionMonths === undefined ? BILLING_ROLLUP_RETENTION_MONTHS : rollup.retentionMonths;
+    const now = rollup.now ?? (() => new Date().toISOString());
+    try {
+        const nowMs = Date.parse(now());
+        if (batch > 0) {
+            const reads = await admin.listRequestRollupDays(actor);
+            // A saturated day is a state of that day, not a failure of the read, so it belongs to no run.
+            let previous = previousAttempt(reads.filter((r) => !r.saturated));
+            const due = requestRollupDueDays({
+                nowMs,
+                reads,
+                lookbackDays: rollup.lookbackDays ?? REQUEST_ROLLUP_LOOKBACK_DAYS,
+                settleMs: rollup.settleMs ?? REQUEST_ROLLUP_SETTLE_MS,
+                retryMs: rollup.retryMs ?? REQUEST_ROLLUP_RETRY_MS,
+                horizon: billingRollupHorizon(nowMs, months),
+            });
+            const take = due.slice(0, batch);
+            out.due = due.length;
+            out.deferred = due.length - take.length;
+            // One day at a time: each is one Analytics Engine query, and the account's query budget
+            // is shared with every observability read.
+            const saturated = [];
+            for (const day of take) {
+                try {
+                    const counts = await withinDeadline(rollup.read(day), rollup.readTimeoutMs ?? REQUEST_ROLLUP_READ_TIMEOUT_MS, 'the request rollup read');
+                    out.recorded += (await admin.recordRequestDay(actor, { day, attemptedAt: now(), counts })).recorded;
+                    out.read += 1;
+                    // A read that succeeds ends any run, so a failure later in this pass starts a new one.
+                    previous = NO_ATTEMPT;
+                }
+                catch (err) {
+                    const error = message(err);
+                    const attemptedAt = now();
+                    if (isRequestRollupSaturated(err)) {
+                        // A standing state of this day, not a failed read: kept visible on the day's record,
+                        // never read again, and no reason to stop, so the days after it are still rolled up.
+                        out.saturated += 1;
+                        saturated.push(day);
+                        await admin.recordRequestDay(actor, { day, attemptedAt, error, saturated: true }).catch(() => undefined);
+                        continue;
+                    }
+                    out.failed += 1;
+                    const { run, repeated } = extendRun(previous, error, attemptedAt);
+                    out.streak = run;
+                    if (repeated)
+                        out.repeated += 1;
+                    else
+                        report.errors.push({ kind: 'request-rollup', id: day, error });
+                    // The failure is the day's attempt, so the day waits `retryMs` rather than taking a slot
+                    // on every pass. If even this write fails, the day is simply due again next pass.
+                    await admin
+                        .recordRequestDay(actor, { day, attemptedAt, error, failingSince: run.since, failingCount: run.failures })
+                        .catch(() => undefined);
+                    // The next day would most likely fail the same way, and each read is a fleet-wide query
+                    // against the account's shared budget: one failure ends the pass's reads.
+                    break;
+                }
+            }
+            // No read failed this pass and none succeeded (a success resets `previous`): a run the ledger
+            // holds still stands, though this pass had nothing due to retry. It is reported as it is,
+            // with no attempt added and nothing pushed to the digest, which mailed it when it began.
+            if (out.streak === null && previous.error !== null && previous.since !== null) {
+                out.streak = { error: previous.error, since: previous.since, failures: previous.failures };
+            }
+            // Each saturated day reaches the digest once, on the pass that found it: it is not read again.
+            out.saturatedDays = saturated;
+            if (saturated.length > 0) {
+                report.errors.push({
+                    kind: 'request-rollup',
+                    id: 'saturated',
+                    error: `${saturated.length} day(s) hold more groups than one read answers and are kept unread: ${saturated.join(', ')}`,
+                });
+            }
+        }
+        if (typeof admin.pruneRequestDays === 'function') {
+            out.pruned = await admin.pruneRequestDays(actor, rollup.pruneBatch ?? REQUEST_ROLLUP_PRUNE_BATCH, months);
+        }
+    }
+    catch (err) {
+        // A directory read or the prune failed: whatever days were recorded stay recorded, and the
+        // next pass finds the rest due.
+        report.errors.push({ kind: 'request-rollup', id: 'directory', error: message(err) });
+    }
+    return out;
+}
+const DAY_MS = 86_400_000;
+/**
+ * One pass of the meter-snapshot phase (#2501): take the snapshot of the UTC day the pass runs in,
+ * unless the day already has one, then prune past retention. A failed attempt writes nothing of the
+ * day and is recorded, and the next pass of the day tries again. A failure that repeats the attempt
+ * before it, across days too, is a standing one: reported when its run began, and not again until
+ * a snapshot is taken.
+ */
+async function sweepMeterSnapshot(admin, actor, snapshot, report) {
+    const out = { day: null, taken: false, tenants: 0, failed: 0, repeated: 0, streak: null, pruned: 0 };
+    const months = snapshot.retentionMonths === undefined ? BILLING_ROLLUP_RETENTION_MONTHS : snapshot.retentionMonths;
+    const now = snapshot.now ?? (() => new Date().toISOString());
+    try {
+        const at = new Date(now()).toISOString();
+        const day = at.slice(0, 10);
+        out.day = day;
+        // The attempt before this pass is today's, or yesterday's when today has none: one a day at the
+        // least while the phase runs, so two days hold it.
+        const takes = await admin.listMeterDayTakes(actor, { since: utcDayOf(Date.parse(at) - DAY_MS), until: day });
+        // A day already taken is not tried again, so a later pass of the day can never report a failure
+        // about a snapshot that stands. The directory checks again inside its own transaction, which is
+        // what settles two passes racing for the same day.
+        if (!takes.some((t) => t.day === day && t.takenAt !== null)) {
+            try {
+                const taken = await withinDeadline(admin.recordMeterDay(actor, { at }), snapshot.timeoutMs ?? METER_SNAPSHOT_TIMEOUT_MS, 'the meter snapshot');
+                out.taken = taken.taken;
+                out.tenants = taken.tenants;
+            }
+            catch (err) {
+                const error = message(err);
+                out.failed = 1;
+                const { run, repeated } = extendRun(previousAttempt(takes), error, at);
+                out.streak = run;
+                if (repeated)
+                    out.repeated = 1;
+                else
+                    report.errors.push({ kind: 'meter-snapshot', id: day, error });
+                // The attempt's own instant, so its record lands on its day even when the deadline ran past
+                // midnight. If even this write fails, the next pass simply tries again.
+                await admin
+                    .recordMeterDay(actor, { at, error, failingSince: run.since, failingCount: run.failures })
+                    .catch(() => undefined);
+            }
+        }
+        if (typeof admin.pruneMeterDays === 'function') {
+            out.pruned = await admin.pruneMeterDays(actor, snapshot.pruneBatch ?? METER_DAYS_PRUNE_BATCH, months);
+        }
+    }
+    catch (err) {
+        // A directory read or the prune failed: a snapshot already taken stays taken, and the next pass
+        // tries the day again if it was not.
+        report.errors.push({ kind: 'meter-snapshot', id: 'directory', error: message(err) });
+    }
+    return out;
+}
 const pushCapped = (bucket, id) => {
     bucket.count += 1;
     if (bucket.scopeIds.length < ORPHAN_ATTACHMENT_REPORTED_IDS)