@substrat-run/control-plane-api 0.159.0 → 0.161.0

package/dist/cf-observability.js

@@ -4,7 +4,8 @@ import { cachedSource, cutOverSource } from './aggregate-source.js';
 import { stableDeploymentRefFor } from './deploy.js';
 import { aggregateReads } from './aggregate-reads.js';
 import { serviceFamilyPattern } from './service-family.js';
-import { FIELD_COVERAGE_ID_FIELD, ROUTER_SCRIPT_NAMES } from '@substrat-run/contracts';
+import { FIELD_COVERAGE_ID_FIELD, ROUTER_REQUEST_POINT, ROUTER_SCRIPT_NAMES, utcDay } from '@substrat-run/contracts';
+import { RequestRollupSaturated } from '@substrat-run/kernel';
 import { readFieldCoverage } from './field-coverage-read.js';
 import { TENANT_METRICS_LIMIT, REQUEST_FACET_KEYS, INVOCATION_KINDS, invocationKindOf, outboundCallsOf } from './observability.js';
 /**
@@ -192,6 +193,16 @@ function requestRecordOf(e) {
  */
 const MAX_CORRELATED_INVOCATIONS = 40;
 const MAX_LINES_PER_INVOCATION = 20;
+/**
+ * How a service-grain read (`queryEvents`) is bounded while it fills its page after the wire line
+ * is left out: at most this many rounds of queries (the first round is one per service), and at
+ * most `MAX_FILL_QUERIES` queries past that first round in all, whatever the number of services.
+ * A page loses at most about a third to the wire line, so one more query nearly always fills it;
+ * the bounds keep a window of almost nothing else, or a read across many services, from costing
+ * more than a few.
+ */
+const MAX_FILL_ROUNDS = 4;
+const MAX_FILL_QUERIES = 8;
 /**
  * #1877: the groups one telemetry answer carries, and how many pages of them a span may
  * take. Past the cap a span is refused rather than cached in part.
@@ -442,6 +453,38 @@ function withoutOutputFields(e) {
     const { outputFields: _report, [FIELD_COVERAGE_ID_FIELD]: _id, ...rest } = source;
     return { ...e, source: rest };
 }
+/**
+ * The events of a telemetry `events` answer, as every reader in this module receives them.
+ *
+ * The list is unwrapped: the view has nested it one level deeper across API revisions, so
+ * both shapes are accepted rather than pinning whichever was current at writing. And the
+ * platform's wire-signing line (`substrat: 'wire'`, RFC wire-signing §8) is left out, here
+ * and only here, so no reader can return one: it is the platform's counter, it carries a
+ * refusal's reason, and it shares a request's id, which is how a tenant's view finds the
+ * lines it shows. No reader today needs it; one that does (the readiness report, RFC §10
+ * follow-on 4) is to read it through a query of its own that opts in by name.
+ *
+ * The line is left out AFTER the backend spent its `limit` on it, because no filter of the
+ * telemetry API is known to keep the lines that lack the key it negates. So beside the events go
+ * the size of what the backend returned (`fetched`) and its oldest timestamp, never a line of it:
+ * a reader judges a full page (more may exist) by those, and asks again for what the filter took
+ * (`queryEvents`, and the tenant view's per-request read).
+ */
+function telemetryPage(result) {
+    const outer = result?.events;
+    const all = (Array.isArray(outer) ? outer : (outer?.events ?? []));
+    let oldest = null;
+    for (const e of all) {
+        const t = e['timestamp'];
+        if (typeof t === 'number' && (oldest === null || t < oldest))
+            oldest = t;
+    }
+    return {
+        events: all.filter((e) => (e['source'] ?? {})['substrat'] !== 'wire'),
+        fetched: all.length,
+        oldest,
+    };
+}
 /**
  * One backend event → the seam's neutral `RecentLogEvent`.
  *
@@ -698,21 +741,22 @@ export function createCfObservabilityReader(opts) {
         const pending = inFlight.get(key);
         if (pending)
             return pending;
+        const p = ROUTER_REQUEST_POINT;
         const sql = `
       SELECT
-        blob1 AS vertical,
-        blob2 AS scopeId,
-        blob3 AS surface,
-        double2 AS status,
-        blob6 AS operation,
-        blob7 AS problemCode,
-        blob8 AS principalKind,
-        blob9 AS level,
+        ${p.vertical} AS vertical,
+        ${p.scopeId} AS scopeId,
+        ${p.surface} AS surface,
+        ${p.status} AS status,
+        ${p.operation} AS operation,
+        ${p.problemCode} AS problemCode,
+        ${p.principalKind} AS principalKind,
+        ${p.level} AS level,
         toStartOfInterval(timestamp, INTERVAL '${grainSeconds}' SECOND) AS bucket,
         sum(_sample_interval) AS requests,
         max(_sample_interval) AS maxInterval
       FROM ${aeDataset(dataset)}
-      WHERE index1 = ${aeLiteral(tenantId)}
+      WHERE ${p.tenantId} = ${aeLiteral(tenantId)}
         AND timestamp >= toDateTime(${Math.floor(q.from / 1000)})
         AND timestamp < toDateTime(${Math.ceil(q.to / 1000)})
       GROUP BY vertical, scopeId, surface, status, operation, problemCode, principalKind, level, bucket
@@ -909,14 +953,7 @@ export function createCfObservabilityReader(opts) {
             // single-valued equality, and an OR across script names is not a filter this
             // API offers. The queries are independent, so they run concurrently and the
             // merge (not the backend) enforces the overall `limit`.
-            if (services && services.length > 1) {
-                const pages = await Promise.all(services.map((s) => queryEvents(s, level, search, hours, limit)));
-                return pages
-                    .flat()
-                    .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
-                    .slice(0, limit);
-            }
-            return queryEvents(services?.[0], level, search, hours, limit);
+            return queryEvents(services && services.length > 1 ? services : [services?.[0]], level, search, hours, limit);
         },
         // Present only when the caller named the router's dataset: with no dataset there is
         // no honest answer, and the route's 501 says so. See `routerDataset`. The bucketed
@@ -1057,33 +1094,32 @@ export function createCfObservabilityReader(opts) {
         // would answer two rows whose p95s no caller can combine. Vertical reads null there.
         const byScope = input.grain === 'scope';
         const window = resolveObservabilityWindow(input);
+        // Blob/double positions are the router's published shape, `ROUTER_REQUEST_POINT`: it only
+        // ever grows, never reorders, which is what lets a reader name the ordinal it wants. The
+        // connector-call dataset's ordinals are published beside these, on
+        // `queryConnectorCallsSeries` below (#1691) — a different dataset, never this one.
+        const p = ROUTER_REQUEST_POINT;
         const where = [
-            `index1 = ${aeLiteral(input.tenantId)}`,
+            `${p.tenantId} = ${aeLiteral(input.tenantId)}`,
             `timestamp >= toDateTime(${Math.ceil(Date.parse(window.since) / 1000)})`,
             `timestamp < toDateTime(${Math.ceil(Date.parse(window.until) / 1000)})`,
         ];
         if (input.scopeId)
-            where.push(`blob2 = ${aeLiteral(input.scopeId)}`);
+            where.push(`${p.scopeId} = ${aeLiteral(input.scopeId)}`);
         if (input.vertical)
-            where.push(`blob1 = ${aeLiteral(input.vertical)}`);
-        // Blob/double positions are the router's published shape (`apps/router/src/worker.ts`
-        // `record`): index1 tenant; blob1 vertical, blob2 scope, blob3 surface, blob4 status
-        // class; double1 duration ms, double2 status. That shape only ever grows, never
-        // reorders — which is what lets these ordinals be written down here at all. The
-        // connector-call dataset's ordinals are published beside these, on
-        // `queryConnectorCallsSeries` below (#1691) — a different dataset, never this one.
+            where.push(`${p.vertical} = ${aeLiteral(input.vertical)}`);
         const sql = `
       SELECT
-        blob2 AS scopeId,
-        ${byScope ? "'' AS vertical" : 'blob1 AS vertical'},
-        ${byScope ? "'' AS surface" : 'blob3 AS surface'},
+        ${p.scopeId} AS scopeId,
+        ${byScope ? "'' AS vertical" : `${p.vertical} AS vertical`},
+        ${byScope ? "'' AS surface" : `${p.surface} AS surface`},
         sum(_sample_interval) AS requests,
-        sum(if(blob4 = '5xx', _sample_interval, 0)) AS errors,
-        sum(if(blob4 = '2xx', _sample_interval, 0)) AS class2xx,
-        sum(if(blob4 = '3xx', _sample_interval, 0)) AS class3xx,
-        sum(if(blob4 = '4xx', _sample_interval, 0)) AS class4xx,
-        quantileWeighted(0.5)(double1, _sample_interval) AS durationP50,
-        quantileWeighted(0.95)(double1, _sample_interval) AS durationP95
+        sum(if(${p.statusClass} = '5xx', _sample_interval, 0)) AS errors,
+        sum(if(${p.statusClass} = '2xx', _sample_interval, 0)) AS class2xx,
+        sum(if(${p.statusClass} = '3xx', _sample_interval, 0)) AS class3xx,
+        sum(if(${p.statusClass} = '4xx', _sample_interval, 0)) AS class4xx,
+        quantileWeighted(0.5)(${p.durationMs}, _sample_interval) AS durationP50,
+        quantileWeighted(0.95)(${p.durationMs}, _sample_interval) AS durationP95
       FROM ${aeDataset(dataset)}
       WHERE ${where.join(' AND ')}
       GROUP BY ${byScope ? 'scopeId' : 'scopeId, vertical, surface'}
@@ -1130,22 +1166,23 @@ export function createCfObservabilityReader(opts) {
         const window = resolveObservabilityWindow(input);
         const bucketMinutes = observabilityBucketMinutes(window);
         const scopes = input.scopeIds.map((s) => aeLiteral(s)).join(', ');
+        const p = ROUTER_REQUEST_POINT;
         const sql = `
       SELECT
-        blob2 AS scopeId,
+        ${p.scopeId} AS scopeId,
         toStartOfInterval(timestamp, INTERVAL '${bucketMinutes}' MINUTE) AS start,
         sum(_sample_interval) AS requests,
-        sum(if(blob4 = '5xx', _sample_interval, 0)) AS errors,
-        sum(if(blob4 = '2xx', _sample_interval, 0)) AS class2xx,
-        sum(if(blob4 = '3xx', _sample_interval, 0)) AS class3xx,
-        sum(if(blob4 = '4xx', _sample_interval, 0)) AS class4xx,
-        quantileWeighted(0.5)(double1, _sample_interval) AS durationP50,
-        quantileWeighted(0.95)(double1, _sample_interval) AS durationP95
+        sum(if(${p.statusClass} = '5xx', _sample_interval, 0)) AS errors,
+        sum(if(${p.statusClass} = '2xx', _sample_interval, 0)) AS class2xx,
+        sum(if(${p.statusClass} = '3xx', _sample_interval, 0)) AS class3xx,
+        sum(if(${p.statusClass} = '4xx', _sample_interval, 0)) AS class4xx,
+        quantileWeighted(0.5)(${p.durationMs}, _sample_interval) AS durationP50,
+        quantileWeighted(0.95)(${p.durationMs}, _sample_interval) AS durationP95
       FROM ${aeDataset(dataset)}
-      WHERE index1 = ${aeLiteral(input.tenantId)}
+      WHERE ${p.tenantId} = ${aeLiteral(input.tenantId)}
         AND timestamp >= toDateTime(${Math.ceil(Date.parse(window.since) / 1000)})
         AND timestamp < toDateTime(${Math.ceil(Date.parse(window.until) / 1000)})
-        AND blob2 IN (${scopes})
+        AND ${p.scopeId} IN (${scopes})
       GROUP BY scopeId, start
       ORDER BY start ASC
       LIMIT ${SERIES_ROW_LIMIT}
@@ -1286,14 +1323,8 @@ export function createCfObservabilityReader(opts) {
         });
     }
     /** One Analytics Engine SQL read, returning the rows as the API shaped them. */
-    async function analyticsEngineSql(sql) {
-        const res = await fetch(`https://api.cloudflare.com/client/v4/accounts/${opts.accountId}/analytics_engine/sql`, { method: 'POST', headers: { authorization: `Bearer ${opts.apiToken}` }, body: sql });
-        const text = await res.text();
-        if (!res.ok) {
-            throw new Error(`Cloudflare Analytics Engine query failed: HTTP ${res.status} ${text.slice(0, 200)}`);
-        }
-        const json = JSON.parse(text);
-        return json.data ?? [];
+    function analyticsEngineSql(sql) {
+        return analyticsEngineQuery(opts, sql);
     }
     /**
      * One tenant's log events, in two phases (§4.3).
@@ -1330,6 +1361,23 @@ export function createCfObservabilityReader(opts) {
             from: input.since ? Date.parse(input.since) : to - input.hours * 3_600_000,
             to,
         };
+        // #2521: set when a page of this tenant's lines came back FULL from the backend and held fewer
+        // than the read wanted once a line was taken out (`telemetryPage` leaves the wire line out after
+        // the backend has spent a slot on it), so lines may lie behind the cut. It is the service
+        // grain's `incomplete`: a short page is then not "the window held no more". `wanted` is what the
+        // read keeps of a full page: all of it, except where it over-fetched a slot to give back (the
+        // per-request read, below), which a single filtered line leaves whole. Set by this tenant's own
+        // queries only: an account-wide query's filtered line is not necessarily this tenant's.
+        let incomplete = false;
+        const noteCut = (page, asked, wanted = asked) => {
+            if (page.fetched >= asked && page.events.length < wanted)
+                incomplete = true;
+        };
+        const readTenant = async (filters, asked) => {
+            const page = await queryRawPage(filters, timeframe, asked);
+            noteCut(page, asked);
+            return page.events;
+        };
         const base = [
             { key: 'tenantId', operation: 'eq', type: 'string', value: input.tenantId },
             { key: 'substrat', operation: 'eq', type: 'string', value: 'invocation' },
@@ -1400,20 +1448,21 @@ export function createCfObservabilityReader(opts) {
         // line from another of the tenant's apps.
         const moduleLines = input.vertical && !input.scopeId
             ? Promise.resolve([])
-            : queryRaw([...moduleLineFilters(input), ...serviceFilter(input.services)], timeframe, input.limit);
+            : readTenant([...moduleLineFilters(input), ...serviceFilter(input.services)], input.limit);
         // A template names `ctx.log` lines only: an invocation line has none.
         if (input.template !== undefined) {
-            return (await moduleLines)
+            const lines = (await moduleLines)
                 .map((e) => describeModuleLine(projectEvent(e)))
                 .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
                 .slice(0, input.limit);
+            return { events: lines, incomplete };
         }
         const [stamped, errorLines] = isErrorRead
             ? await Promise.all([
                 Promise.all([
-                    queryRaw([...base, { key: 'status', operation: 'eq', type: 'number', value: 500 }], timeframe, phaseOneLimit),
-                    queryRaw([...base, { key: 'status', operation: 'gte', type: 'number', value: CAPABILITY_ABSENT + 1 }], timeframe, phaseOneLimit),
-                    queryRaw([...base, { key: 'threw', operation: 'eq', type: 'boolean', value: true }], timeframe, phaseOneLimit),
+                    readTenant([...base, { key: 'status', operation: 'eq', type: 'number', value: 500 }], phaseOneLimit),
+                    readTenant([...base, { key: 'status', operation: 'gte', type: 'number', value: CAPABILITY_ABSENT + 1 }], phaseOneLimit),
+                    readTenant([...base, { key: 'threw', operation: 'eq', type: 'boolean', value: true }], phaseOneLimit),
                     // Newest first across the pages, not page order: the correlation cap is spent
                     // in this order, and a busy `= 500` page must not crowd out a newer 502 or
                     // escape just because its query was listed first.
@@ -1423,7 +1472,7 @@ export function createCfObservabilityReader(opts) {
                 // and `ownsInvocation` below still admits nothing without a stamped line.
                 [{ key: '$metadata.level', operation: 'eq', type: 'string', value: 'error' }, ...serviceFilter(input.services)], timeframe, phaseOneLimit),
             ])
-            : [await queryRaw(base, timeframe, phaseOneLimit), []];
+            : [await readTenant(base, phaseOneLimit), []];
         const trusted = invocationIds(stamped);
         const trustedIds = new Set(trusted);
         const candidates = invocationIds(errorLines).filter((id) => !trustedIds.has(id));
@@ -1441,13 +1490,20 @@ export function createCfObservabilityReader(opts) {
         // `limit` (the route bounds that at 200).
         const linesPerInvocation = input.invocationId !== undefined ? Math.min(input.limit, 200) : MAX_LINES_PER_INVOCATION;
         const sibling = await Promise.all(requestIds.map(async (id) => {
-            const events = await queryRaw([{ key: '$metadata.requestId', operation: 'eq', type: 'string', value: id }, ...serviceFilter(input.services)], timeframe, linesPerInvocation);
+            // One more than the budget, then cut to it: a request writes at most one wire line (RFC
+            // wire-signing §8), which `telemetryPage` leaves out after the backend spent a slot on it.
+            // So the page holds exactly the lines it would if that line did not exist.
+            const page = await queryRawPage([{ key: '$metadata.requestId', operation: 'eq', type: 'string', value: id }, ...serviceFilter(input.services)], timeframe, linesPerInvocation + 1);
+            const events = page.events.slice(0, linesPerInvocation);
             // An account-wide candidate earns its place only by producing this tenant's
             // stamped line. No stamped line, or somebody else's, and the whole invocation is
             // dropped — a line shown to the wrong tenant is exactly the leak the tenant grain
             // exists to prevent, so the conservative direction is the only allowed one.
             if (!trustedIds.has(id) && !events.some((e) => ownsInvocation(e, input)))
                 return [];
+            // Only an invocation this tenant owns can be incomplete for it. The slot over-fetched above
+            // gives one filtered line back; a full page that lost more than that is short of its share.
+            noteCut(page, linesPerInvocation + 1, linesPerInvocation);
             return events;
         }));
         // Merge, de-duplicate, drop the router's own access log, then apply the caller's
@@ -1470,7 +1526,7 @@ export function createCfObservabilityReader(opts) {
         const runtimeOf = foldRuntimeRecords([...byId.values()]);
         const level = input.level?.toLowerCase();
         const search = input.search;
-        return runtimeOf.kept
+        const events = runtimeOf.kept
             .map((e) => withRuntimeTimings(projectEvent(e), runtimeOf.byRequest))
             .map((e) => describeModuleLine(describeInvocation(e)))
             // A plain comparison is enough only because `describeInvocation` ran first: a stamped
@@ -1483,6 +1539,7 @@ export function createCfObservabilityReader(opts) {
             .filter((e) => (search ? (e.message ?? '').includes(search) : true))
             .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
             .slice(0, input.limit);
+        return { events, incomplete };
     }
     /** The de-duplicated, order-preserving request ids of a page of events. */
     function invocationIds(events) {
@@ -1545,6 +1602,10 @@ export function createCfObservabilityReader(opts) {
     // searches the same one, so this takes the instants rather than a duration it would
     // have to re-anchor to a `Date.now()` of its own.
     timeframe, limit) {
+        return (await queryRawPage(filters, timeframe, limit)).events;
+    }
+    /** {@link queryRaw}, with the size of the page the backend returned (see `telemetryPage`). */
+    async function queryRawPage(filters, timeframe, limit) {
         const res = await authed(`https://api.cloudflare.com/client/v4/accounts/${opts.accountId}/workers/observability/telemetry/query`, {
             queryId: 'substrat-tenant-logs',
             view: 'events',
@@ -1558,14 +1619,14 @@ export function createCfObservabilityReader(opts) {
         const json = JSON.parse(text);
         if (json.success === false)
             throw telemetryFailure(502, text);
-        const outer = json.result?.events;
-        return (Array.isArray(outer) ? outer : (outer?.events ?? []));
+        return telemetryPage(json.result);
     }
     /** Read a complete sampled span, or refuse it; a truncated page must never mean zero fields. */
     async function queryAllFieldEvents(filters, timeframe, depth = 0) {
-        const page = await queryRaw(filters, timeframe, 2000);
-        if (page.length < 2000)
-            return page;
+        // Full is judged on what the backend returned, not on what is left once a line is filtered out.
+        const page = await queryRawPage(filters, timeframe, 2000);
+        if (page.fetched < 2000)
+            return page.events;
         if (depth >= 12 || timeframe.to - timeframe.from <= 1000) {
             throw new ControlPlaneError(503, 'field coverage window is too busy — choose a shorter window');
         }
@@ -1576,38 +1637,103 @@ export function createCfObservabilityReader(opts) {
         ]);
         return [...left, ...right];
     }
-    /** One telemetry query — narrowed to a single service, or to none (the fleet view). */
-    async function queryEvents(service, level, search, hours, limit) {
+    /**
+     * The service-grain read: one walk per service, or one over none (the fleet view), merged
+     * newest first and cut to `limit`, and whether it stopped short (`RecentLogs.incomplete`).
+     */
+    async function queryEvents(services, level, search, hours, limit) {
         const to = Date.now();
-        const filters = [];
-        // `$metadata.service` is the script name in Workers Logs events — the field
-        // Cloudflare's own query examples filter on.
-        if (service) {
-            filters.push({ key: '$metadata.service', operation: 'eq', type: 'string', value: service });
-        }
-        if (level) {
-            filters.push({ key: '$metadata.level', operation: 'eq', type: 'string', value: level });
-        }
-        if (search) {
-            filters.push({ key: '$metadata.message', operation: 'includes', type: 'string', value: search });
-        }
-        const res = await authed(`https://api.cloudflare.com/client/v4/accounts/${opts.accountId}/workers/observability/telemetry/query`, {
-            queryId: 'substrat-recent-logs',
-            view: 'events',
-            timeframe: { from: to - hours * 3_600_000, to },
-            parameters: { datasets: ['cloudflare-workers'], filters, limit },
-            limit,
-        });
-        const json = (await res.json());
-        if (!res.ok || json.success === false) {
-            const message = json.errors?.map((e) => e.message).join('; ') || `HTTP ${res.status}`;
-            throw new Error(`Cloudflare telemetry query failed: ${message}`);
+        const from = to - hours * 3_600_000;
+        const filtersFor = (service) => {
+            const filters = [];
+            // `$metadata.service` is the script name in Workers Logs events — the field
+            // Cloudflare's own query examples filter on.
+            if (service) {
+                filters.push({ key: '$metadata.service', operation: 'eq', type: 'string', value: service });
+            }
+            if (level) {
+                filters.push({ key: '$metadata.level', operation: 'eq', type: 'string', value: level });
+            }
+            if (search) {
+                filters.push({ key: '$metadata.message', operation: 'includes', type: 'string', value: search });
+            }
+            return filters;
+        };
+        const page = async (filters, timeframe) => {
+            const res = await authed(`https://api.cloudflare.com/client/v4/accounts/${opts.accountId}/workers/observability/telemetry/query`, {
+                queryId: 'substrat-recent-logs',
+                view: 'events',
+                timeframe,
+                parameters: { datasets: ['cloudflare-workers'], filters, limit },
+                limit,
+            });
+            const json = (await res.json());
+            if (!res.ok || json.success === false) {
+                const message = json.errors?.map((e) => e.message).join('; ') || `HTTP ${res.status}`;
+                throw new Error(`Cloudflare telemetry query failed: ${message}`);
+            }
+            return telemetryPage(json.result);
+        };
+        // Filled to `limit` when more exist. A full page the wire line was taken out of (in `dual`,
+        // up to a third of one: a request writes its runtime record, its stamp and the wire line) may
+        // be followed by the page before its oldest line: its next window ends just after that oldest
+        // instant, and the event id folds the overlap. A walk is read again only while its next page
+        // could still reach the merged answer, so a read across many services pays for the fill only
+        // where a service's page decides the answer's tail, and never more than `MAX_FILL_QUERIES`.
+        //
+        // A read that stops while a walk could still hold a line for the answer says so (`incomplete`):
+        // a walk the bounds left unread, or a full page all of one instant, which no earlier window
+        // can cut past.
+        const walks = services.map((service) => ({ filters: filtersFor(service), until: to, more: false, stuck: false, oldest: 0 }));
+        const kept = new Map();
+        const read = async (walk) => {
+            const { events, fetched, oldest } = await page(walk.filters, { from, to: walk.until });
+            for (const e of events) {
+                const metadata = (e['$metadata'] ?? {});
+                kept.set(typeof metadata['id'] === 'string' ? metadata['id'] : JSON.stringify(e), e);
+            }
+            // More may exist only behind a full page, and only if an earlier window can be cut: a page
+            // all of one instant cannot be, so the walk stops there, stuck rather than spent.
+            walk.more = fetched >= limit && oldest !== null && oldest + 1 < walk.until;
+            walk.stuck = fetched >= limit && !walk.more;
+            if (oldest !== null)
+                walk.oldest = oldest;
+            if (walk.more)
+                walk.until = oldest + 1;
+        };
+        const answer = () => [...kept.values()].sort((a, b) => rawTime(b) - rawTime(a)).slice(0, limit);
+        // Lines older than this may have a newer one behind a walk the budget left unread.
+        let floor = -Infinity;
+        let incomplete = false;
+        let budget = MAX_FILL_QUERIES;
+        let pending = walks;
+        for (let round = 0; pending.length > 0; round++) {
+            await Promise.all(pending.map(read));
+            const merged = answer();
+            // A walk's unread lines are no newer than its oldest read one, so it can change the answer
+            // only while that line is newer than the answer's last (or the answer is short). Not "as
+            // new": the walk holding the answer's last line could add only lines of that same instant,
+            // whose order against it is arbitrary anyway, and every full page would cost a second query.
+            const cutoff = merged.length < limit ? -Infinity : rawTime(merged[merged.length - 1]);
+            const reaches = (w) => w.oldest > cutoff;
+            const wanted = walks.filter((w) => w.more && reaches(w)).sort((a, b) => b.oldest - a.oldest);
+            const roundsLeft = round + 1 < MAX_FILL_ROUNDS;
+            pending = roundsLeft ? wanted.slice(0, budget) : [];
+            budget -= pending.length;
+            const unread = wanted.slice(pending.length);
+            // Left unread by the budget while others are read on: their older lines would sit beside
+            // lines this walk may still hold, so the answer stops at its oldest and stays a true prefix.
+            // A walk that ran out of rounds is not a floor: it was read as far as every other, and a
+            // floor from it would let one service whose pages are all wire lines empty the answer.
+            if (roundsLeft)
+                for (const w of unread)
+                    floor = Math.max(floor, w.oldest);
+            incomplete = unread.length > 0 || walks.some((w) => w.stuck && reaches(w));
         }
-        // The events view has nested the list one level deeper across API revisions —
-        // accept both rather than pinning to whichever shape was current at writing.
-        const outer = json.result?.events;
-        const events = (Array.isArray(outer) ? outer : (outer?.events ?? []));
-        return events.map((e) => projectEvent(e));
+        const events = answer()
+            .filter((e) => rawTime(e) >= floor)
+            .map((e) => projectEvent(e));
+        return { events, incomplete };
     }
     /**
      * Outbound `fetch` spans for one service (#859).
@@ -1651,8 +1777,7 @@ export function createCfObservabilityReader(opts) {
             const message = json.errors?.map((e) => e.message).join('; ') || `HTTP ${res.status}`;
             throw new Error(`Cloudflare span query failed: ${message}`);
         }
-        const outer = json.result?.events;
-        const events = (Array.isArray(outer) ? outer : (outer?.events ?? []));
+        const { events, fetched } = telemetryPage(json.result);
         const spans = events.map((e) => {
             const source = (e['source'] ?? {});
             const metadata = (e['$metadata'] ?? {});
@@ -1682,8 +1807,88 @@ export function createCfObservabilityReader(opts) {
         });
         // Row-cap detection: a full page means there may be more. Reported rather than
         // swallowed — an egress report that silently drops hosts is worse than none (#859).
-        return { spans, truncated: events.length >= limit };
+        return { spans, truncated: fetched >= limit };
+    }
+}
+/** One Analytics Engine SQL read, returning the rows as the API shaped them. */
+async function analyticsEngineQuery(opts, sql) {
+    const res = await fetch(`https://api.cloudflare.com/client/v4/accounts/${opts.accountId}/analytics_engine/sql`, { method: 'POST', headers: { authorization: `Bearer ${opts.apiToken}` }, body: sql });
+    const text = await res.text();
+    if (!res.ok) {
+        throw new Error(`Cloudflare Analytics Engine query failed: HTTP ${res.status} ${text.slice(0, 200)}`);
     }
+    const json = JSON.parse(text);
+    return json.data ?? [];
+}
+/**
+ * The most rows one day's rollup query may answer (#2501): one per (tenant, vertical, scope) that
+ * served a request that day. A day with more is refused as `RequestRollupSaturated`, never kept as
+ * a part of the day standing for the whole. The rollup records such a day as saturated and goes on
+ * with the days after it; reading it would need a paged read, which this is not yet.
+ */
+export const REQUEST_ROLLUP_ROW_LIMIT = 50_000;
+/**
+ * The daily request rollup's reader (#2501): one closed UTC day of the router's request
+ * datapoints, counted per (tenant, vertical, scope). The platform sweep's request-rollup phase
+ * calls it, never a route.
+ *
+ * It reads EVERY tenant at once, which is why it is not a member of `ObservabilityReader`: that
+ * seam answers routes, and keeps a fleet-wide read one forgotten parameter away by having no
+ * all-tenants spelling. One query per day for the whole fleet is what keeps the rollup to a
+ * handful of queries a pass.
+ *
+ * Counts are `sum(_sample_interval)`, never `count()`, for `queryTenantMetrics`'s reason: Analytics
+ * Engine samples under load, so each count is an estimate and `sampled` says whether any of the
+ * day's datapoints stood for more than one request. The ordinals are the router's published shape
+ * (`ROUTER_REQUEST_POINT`). Beside the total it counts each status class from `statusClass` and
+ * the 429s from `status`, weighted the same way, so the split is kept while the datapoints last.
+ */
+export function createRouterRequestDayReader(opts) {
+    return async (day) => {
+        if (!utcDay.safeParse(day).success) {
+            throw new Error(`request rollup: refusing a day that is not a UTC day: ${JSON.stringify(day)}`);
+        }
+        const from = Date.parse(`${day}T00:00:00Z`) / 1000;
+        const p = ROUTER_REQUEST_POINT;
+        const sql = `
+      SELECT
+        ${p.tenantId} AS tenantId,
+        ${p.vertical} AS vertical,
+        ${p.scopeId} AS scopeId,
+        sum(_sample_interval) AS requests,
+        sum(if(${p.statusClass} = '2xx', _sample_interval, 0)) AS class2xx,
+        sum(if(${p.statusClass} = '3xx', _sample_interval, 0)) AS class3xx,
+        sum(if(${p.statusClass} = '4xx', _sample_interval, 0)) AS class4xx,
+        sum(if(${p.statusClass} = '5xx', _sample_interval, 0)) AS class5xx,
+        sum(if(${p.status} = 429, _sample_interval, 0)) AS rateLimited,
+        max(_sample_interval) AS maxInterval
+      FROM ${aeDataset(opts.routerDataset)}
+      WHERE timestamp >= toDateTime(${from})
+        AND timestamp < toDateTime(${from + 86_400})
+      GROUP BY tenantId, vertical, scopeId
+      ORDER BY tenantId, vertical, scopeId
+      LIMIT ${REQUEST_ROLLUP_ROW_LIMIT + 1}
+      FORMAT JSON`;
+        const rows = await analyticsEngineQuery(opts, sql);
+        // One row past the ceiling says the day holds more than one answer carries. An answer of
+        // exactly the ceiling is the whole day.
+        if (rows.length > REQUEST_ROLLUP_ROW_LIMIT) {
+            throw new RequestRollupSaturated(`Cloudflare Analytics Engine request rollup holds more than ${REQUEST_ROLLUP_ROW_LIMIT} groups for ${day}: the counts would be partial`);
+        }
+        const count = (r, key) => Math.round(aeNum(r[key]));
+        return rows.map((r) => ({
+            tenantId: String(r['tenantId'] ?? ''),
+            vertical: blank(r['vertical']),
+            scopeId: String(r['scopeId'] ?? ''),
+            requests: count(r, 'requests'),
+            class2xx: count(r, 'class2xx'),
+            class3xx: count(r, 'class3xx'),
+            class4xx: count(r, 'class4xx'),
+            class5xx: count(r, 'class5xx'),
+            rateLimited: count(r, 'rateLimited'),
+            sampled: aeNum(r['maxInterval']) > 1,
+        }));
+    };
 }
 /**
  * The router dataset's name, checked before it is spliced into SQL. It is an IDENTIFIER,