@substrat-run/control-plane-api 0.124.0 → 0.126.0

package/dist/cf-observability.d.ts

@@ -1,3 +1,4 @@
+import { type CubeStore } from './aggregate-source.js';
 import type { ObservabilityReader } from './observability.js';
 /**
  * The Cloudflare implementation of the observability seam (`observability.ts`) —
@@ -10,6 +11,12 @@ import type { ObservabilityReader } from './observability.js';
  */
 export interface CfObservabilityOptions {
     accountId: string;
+    /**
+     * #1877: where closed blocks of counts are kept. With one, every closed block of an
+     * app's script is counted once and read from here after; without one (a test, a dev
+     * control plane), every read counts its span live.
+     */
+    cubeStore?: CubeStore;
     /**
      * A Cloudflare API token with Account Analytics read (the GraphQL invocations
      * dataset) and Workers Observability read (the telemetry query API). Deliberately

package/dist/cf-observability.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cf-observability.d.ts","sourceRoot":"","sources":["../src/cf-observability.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,mBAAmB,EAapB,MAAM,oBAAoB,CAAC;AAgS5B;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;;;;;;;OAgBG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAgED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,sBAAsB,GAAG,mBAAmB,CAkpC7F"}
+{"version":3,"file":"cf-observability.d.ts","sourceRoot":"","sources":["../src/cf-observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAsD,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE3G,OAAO,KAAK,EACV,mBAAmB,EASpB,MAAM,oBAAoB,CAAC;AAqa5B;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;;;;;;;OAgBG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAgED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,sBAAsB,GAAG,mBAAmB,CAqnC7F"}

package/dist/cf-observability.js

@@ -1,5 +1,8 @@
 import { resolveObservabilityWindow, observabilityBucketMinutes } from './observability-window.js';
-import { TENANT_METRICS_LIMIT, REQUEST_FACET_KEYS, REQUEST_FACET_TOP, LOG_PATTERN_TOP } from './observability.js';
+import { ControlPlaneError } from './client.js';
+import { cachedSource } from './aggregate-source.js';
+import { aggregateReads } from './aggregate-reads.js';
+import { TENANT_METRICS_LIMIT, REQUEST_FACET_KEYS } from './observability.js';
 /**
  * The filters for a tenant's stamped invocation lines (#1746), with the facet filters ANDed
  * on. `omit` leaves one facet's own filter out, which is how a facet is counted "with every
@@ -13,6 +16,7 @@ function requestFilters(scope, omit) {
     const filters = [
         { key: 'tenantId', operation: 'eq', type: 'string', value: scope.tenantId },
         { key: 'substrat', operation: 'eq', type: 'string', value: 'invocation' },
+        ...serviceFilter(scope.services),
     ];
     if (scope.scopeId)
         filters.push({ key: 'scopeId', operation: 'eq', type: 'string', value: scope.scopeId });
@@ -31,6 +35,72 @@ function requestFilters(scope, omit) {
     }
     return filters;
 }
+/** A lowercased ULID, as it ends a per-version script name (`deploymentRefFor`). */
+const SCRIPT_ULID = '[0-9a-hjkmnp-tv-z]{26}';
+/**
+ * #1877: one vertical's script FAMILY, as a filter on `$metadata.service` — the field
+ * Workers Logs indexes every event by. The stem itself is the serving script; `<stem>-<ulid>`
+ * is a per-version script (previews, legacy scopes); `<stem>-eu` / `-us` a jurisdictional one
+ * (K-30).
+ *
+ * Two leaves, both required: `starts_with` is the one the store can narrow a scan on, and the
+ * anchored regex is what makes it exact — another vertical whose stem merely begins with this
+ * one (`ticket0` and `ticket0-crm`) is not in the family.
+ */
+function familyFilter(stem) {
+    const escaped = stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return {
+        kind: 'group',
+        filterCombination: 'and',
+        filters: [
+            { key: '$metadata.service', operation: 'starts_with', type: 'string', value: stem },
+            { key: '$metadata.service', operation: 'regex', type: 'string', value: `^${escaped}(-${SCRIPT_ULID})?(-(eu|us))?$` },
+        ],
+    };
+}
+/**
+ * #1877: the app's script families, as a filter — the scan boundary (without it a tenant
+ * read searched every script in the account for a field inside our own JSON) and a trust
+ * boundary (a line counts only if one of the vertical's own scripts wrote it).
+ *
+ * Absent means the caller did not resolve them, and the read falls back to the tenant filter
+ * alone. An EMPTY list is different — it names no script, so it matches no line — and is
+ * spelled as a filter that cannot match rather than as no filter at all.
+ */
+function serviceFilter(services) {
+    if (services === undefined)
+        return [];
+    if (services.length === 0)
+        return [{ key: '$metadata.service', operation: 'eq', type: 'string', value: '\u0000none' }];
+    const families = services.map(familyFilter);
+    return families.length === 1 ? families : [{ kind: 'group', filterCombination: 'or', filters: families }];
+}
+/**
+ * A telemetry answer that is not a success, as the caller should hear it (#1877).
+ *
+ * Read as TEXT first: a gateway timeout arrives as `error code: 504`, not JSON, and
+ * parsing it first turned a timeout into a JSON syntax error behind a blanket 500. A
+ * timeout is a 504 and a throttle a 503, each saying what happened, so the dashboard
+ * can tell "the log store is slow" from "this is broken".
+ */
+function telemetryFailure(status, body) {
+    let detail = body.trim().slice(0, 200);
+    try {
+        const json = JSON.parse(body);
+        const messages = json.errors?.map((e) => e.message).filter(Boolean).join('; ');
+        if (messages)
+            detail = messages;
+    }
+    catch {
+        // Not JSON — the gateway's own page. The status says enough.
+    }
+    if (status === 504 || status === 524 || status === 408) {
+        return new ControlPlaneError(504, 'the log store timed out answering this query — try a shorter window');
+    }
+    if (status === 429)
+        return new ControlPlaneError(503, 'the log store is busy — try again shortly');
+    return new ControlPlaneError(502, `the log store refused the query (HTTP ${status})${detail ? `: ${detail}` : ''}`);
+}
 /**
  * How many requests a point stands for.
  *
@@ -45,27 +115,6 @@ function countOf(p) {
 function sampled(p) {
     return typeof p.sampleInterval === 'number' && p.sampleInterval > 1;
 }
-/** The value of one group key on a point, or `undefined` when the point has none. */
-function groupValue(p, key) {
-    return p.groups?.find((g) => g.key === key)?.value;
-}
-/** A stamped line's `level`, or `unrecorded` for a line written before it existed. */
-function levelOf(v) {
-    return v === 'info' || v === 'warn' || v === 'error' ? v : 'unrecorded';
-}
-/**
- * The bucket width a series was cut at, from its own timestamps: the smallest gap between
- * two consecutive buckets. The request asks for a bucket COUNT, and the backend may round
- * the width it derives from it, so the answer is read back rather than assumed. A series
- * with fewer than two buckets says nothing about its width; the nominal one stands in.
- */
-function bucketWidth(times, nominal) {
-    const sorted = [...new Set(times)].sort((a, b) => a - b);
-    let width = Infinity;
-    for (let i = 1; i < sorted.length; i++)
-        width = Math.min(width, sorted[i] - sorted[i - 1]);
-    return Number.isFinite(width) && width > 0 ? width : nominal;
-}
 /** The stamped line of a raw event, as the Requests view lists it. */
 function requestRecordOf(e) {
     const source = (e['source'] ?? {});
@@ -105,6 +154,16 @@ function requestRecordOf(e) {
  */
 const MAX_CORRELATED_INVOCATIONS = 40;
 const MAX_LINES_PER_INVOCATION = 20;
+/**
+ * #1877: the groups one telemetry answer carries, and how many pages of them a span may
+ * take. Past the cap a span is refused rather than cached in part.
+ */
+const GROUP_PAGE = 2000;
+const GROUP_PAGE_CAP = 20;
+/** How many times a full span is halved before a single grain is paged instead. */
+const SPLIT_DEPTH_CAP = 6;
+/** A value as text, or null. */
+const str = (v) => (typeof v === 'string' && v !== '' ? v : null);
 /**
  * A 501 is an honest refusal, not a failure: the route exists to say "this version does
  * not declare that capability" (an undeclared owner-seat or configure hook, #1345). The
@@ -221,6 +280,78 @@ function describeModuleLine(e) {
         trigger: str(source['operation']) ?? e.trigger,
     };
 }
+/**
+ * Is this the RUNTIME's own record of an invocation — the event Cloudflare writes for every
+ * call to a Worker (request, response status, outcome, CPU and wall time) — rather than a
+ * line the Worker's code wrote?
+ *
+ * Identified by the event's own type, and by nothing looser. The fold below HIDES these,
+ * so a false positive would hide a line a vertical wrote on purpose, while a false
+ * negative only leaves a duplicate row. A predicate built from shape — "has a request and
+ * no JSON source" — could match a vertical's plain `console.log`, so it is not used.
+ */
+function isRuntimeRecord(e) {
+    const metadata = (e['$metadata'] ?? {});
+    const source = (e['source'] ?? {});
+    return metadata['type'] === 'cf-worker-event' && source['substrat'] === undefined;
+}
+/**
+ * Fold the runtime's invocation records into the stamped lines of the same requests.
+ *
+ * Every request a vertical served writes two events that describe it: the runtime's
+ * record and the stamped invocation line (`invocation-log.ts`). The stamped line says
+ * everything the runtime's does — method, path, status — and the tenant, operation,
+ * problem code and what it emitted besides, so the list showed each request twice. Only
+ * the runtime knows the CPU and wall time, so those move onto the stamped row.
+ *
+ * The runtime's `outcome` moves too — `exception`, `exceededCpu` — so a request the runtime
+ * cut short still reads as cut short on its one row.
+ *
+ * A runtime record whose request has no stamped line on the page is kept, defensively. In
+ * the tenant read that cannot currently happen: a request reaches this list only through
+ * its stamped line, and one that left none (it crashed before the middleware wrote it)
+ * names no tenant and was never attributable to anyone's view in the first place.
+ */
+function foldRuntimeRecords(events) {
+    const requestOf = (e) => {
+        const metadata = (e['$metadata'] ?? {});
+        const workers = (e['$workers'] ?? {});
+        const id = metadata['requestId'] ?? workers['requestId'];
+        return typeof id === 'string' && id !== '' ? id : null;
+    };
+    const stamped = new Set(events
+        .filter((e) => (e['source'] ?? {})['substrat'] === 'invocation')
+        .map(requestOf)
+        .filter((id) => id !== null));
+    const kept = [];
+    const byRequest = new Map();
+    for (const e of events) {
+        const request = requestOf(e);
+        if (request !== null && stamped.has(request) && isRuntimeRecord(e)) {
+            byRequest.set(request, e);
+            continue;
+        }
+        kept.push(e);
+    }
+    return { kept, byRequest };
+}
+/** A stamped row, carrying the CPU and wall time of the runtime record folded into it. */
+function withRuntimeTimings(e, byRequest) {
+    const source = (e.raw?.['source'] ?? {});
+    if (source['substrat'] !== 'invocation' || e.requestId === null)
+        return e;
+    const runtime = byRequest.get(e.requestId);
+    if (!runtime)
+        return e;
+    const workers = (runtime['$workers'] ?? {});
+    const num = (v) => (typeof v === 'number' ? v : null);
+    return {
+        ...e,
+        cpuTimeMs: num(workers['cpuTimeMs']) ?? e.cpuTimeMs,
+        wallTimeMs: num(workers['wallTimeMs']) ?? e.wallTimeMs,
+        outcome: typeof workers['outcome'] === 'string' ? workers['outcome'] : e.outcome,
+    };
+}
 /**
  * One backend event → the seam's neutral `RecentLogEvent`.
  *
@@ -318,6 +449,138 @@ export function createCfObservabilityReader(opts) {
         },
         body: JSON.stringify(body),
     });
+    /**
+     * #1877: the telemetry half of the aggregate seam — ONE script's lines over one span,
+     * grouped by tenant and every facet, cut into the grain. Every tenant the script served
+     * is in the answer, which is what lets the cache fill one block for all of them.
+     */
+    const telemetry = {
+        async requests(q) {
+            const { rows, estimated, complete } = await countGrouped(q, 'invocation', [
+                { type: 'string', value: 'tenantId' },
+                { type: 'string', value: 'scopeId' },
+                { type: 'string', value: 'level' },
+                { type: 'string', value: 'operation' },
+                { type: 'string', value: 'problemCode' },
+                { type: 'string', value: 'principalKind' },
+                { type: 'string', value: 'surface' },
+                { type: 'number', value: 'status' },
+            ]);
+            return {
+                estimated,
+                complete,
+                rows: rows.flatMap(({ bucket, groups, count }) => {
+                    const tenantId = str(groups.get('tenantId'));
+                    if (tenantId === null)
+                        return [];
+                    const status = groups.get('status');
+                    return [{
+                            bucket,
+                            tenantId,
+                            scopeId: str(groups.get('scopeId')),
+                            level: str(groups.get('level')),
+                            operation: str(groups.get('operation')),
+                            problemCode: str(groups.get('problemCode')),
+                            principalKind: str(groups.get('principalKind')),
+                            surface: str(groups.get('surface')),
+                            status: typeof status === 'number' ? status : typeof status === 'string' && status !== '' ? Number(status) : null,
+                            count,
+                        }];
+                }),
+            };
+        },
+        async patterns(q) {
+            const { rows, estimated, complete } = await countGrouped(q, 'log', [
+                { type: 'string', value: 'tenantId' },
+                { type: 'string', value: 'scopeId' },
+                { type: 'string', value: 'template' },
+                { type: 'string', value: 'level' },
+                { type: 'string', value: 'operation' },
+            ]);
+            return {
+                estimated,
+                complete,
+                rows: rows.flatMap(({ bucket, groups, count }) => {
+                    const tenantId = str(groups.get('tenantId'));
+                    const template = str(groups.get('template'));
+                    if (tenantId === null || template === null || template === '')
+                        return [];
+                    return [{ bucket, tenantId, scopeId: str(groups.get('scopeId')), template, level: str(groups.get('level')), operation: str(groups.get('operation')), count }];
+                }),
+            };
+        },
+    };
+    const source = opts.cubeStore ? cachedSource(telemetry, opts.cubeStore) : telemetry;
+    /**
+     * One script family's lines of one kind over a span, counted per grain bucket and grouped
+     * by `groupBys`.
+     *
+     * When one answer is full — as many groups as it can carry — the span is SPLIT in two at a
+     * grain boundary and each half counted on its own, rather than paged. Paging with
+     * `offsetBy` depends on an order the telemetry API does not promise to hold between
+     * requests, so pages could overlap or skip groups; a narrower span is a smaller, complete
+     * answer with no order to trust. Only a single grain that is still full is paged, and a
+     * cube built that way is marked incomplete so the cache never keeps it as final.
+     */
+    async function countGrouped(q, kind, groupBys) {
+        const filters = [
+            ...serviceFilter([q.service]),
+            { key: 'substrat', operation: 'eq', type: 'string', value: kind },
+        ];
+        const once = async (from, to, offsetBy) => {
+            const calc = await queryCalculation(filters, { from, to }, {
+                groupBys,
+                granularity: Math.max(1, Math.round((to - from) / q.grainMs)),
+                chartType: 'timeseries',
+                limit: GROUP_PAGE,
+                ...(offsetBy ? { offsetBy } : {}),
+            });
+            const rows = [];
+            const groupsSeen = new Set();
+            let estimated = false;
+            for (const point of calc.series ?? []) {
+                const at = point.time ? Date.parse(point.time) : NaN;
+                if (!Number.isFinite(at))
+                    continue;
+                // Floored to the grain: the backend cuts its buckets from the span's start, which
+                // is aligned, so this only absorbs a stray millisecond.
+                const bucket = Math.floor(at / q.grainMs) * q.grainMs;
+                for (const d of point.data ?? []) {
+                    const count = countOf(d);
+                    if (count === 0)
+                        continue;
+                    estimated ||= sampled(d);
+                    const groups = new Map((d.groups ?? []).map((g) => [g.key ?? '', g.value]));
+                    groupsSeen.add(JSON.stringify([...groups.entries()]));
+                    rows.push({ bucket, groups, count });
+                }
+            }
+            return { rows, estimated, full: groupsSeen.size >= GROUP_PAGE };
+        };
+        const span = async (from, to, depth) => {
+            const first = await once(from, to);
+            if (!first.full)
+                return { rows: first.rows, estimated: first.estimated, complete: true };
+            const grains = Math.round((to - from) / q.grainMs);
+            if (grains > 1 && depth < SPLIT_DEPTH_CAP) {
+                const mid = from + Math.floor(grains / 2) * q.grainMs;
+                const [a, b] = await Promise.all([span(from, mid, depth + 1), span(mid, to, depth + 1)]);
+                return { rows: [...a.rows, ...b.rows], estimated: a.estimated || b.estimated, complete: a.complete && b.complete };
+            }
+            // One grain and still full: page it, and say the answer cannot be trusted as whole.
+            const rows = [...first.rows];
+            let estimated = first.estimated;
+            for (let page = 1; page < GROUP_PAGE_CAP; page++) {
+                const next = await once(from, to, page * GROUP_PAGE);
+                rows.push(...next.rows);
+                estimated ||= next.estimated;
+                if (!next.full)
+                    break;
+            }
+            return { rows, estimated, complete: false };
+        };
+        return span(q.from, q.to, 0);
+    }
     return {
         absoluteTenantWindows: true,
         async serviceMetrics({ hours }) {
@@ -451,183 +714,9 @@ export function createCfObservabilityReader(opts) {
         async tenantLogs(input) {
             return queryTenantLogs(input);
         },
-        // #1746: the request histogram. One `calculations` query, grouped by level and cut
-        // into the asked number of buckets. Every stamped line in the window is counted, not a
-        // sample of them, which is what makes zooming a narrower re-query rather than a crop.
-        async tenantRequestVolume(input) {
-            const timeframe = { from: input.from, to: input.to };
-            const calc = await queryCalculation(requestFilters(input), timeframe, {
-                groupBy: { type: 'string', value: 'level' },
-                granularity: input.buckets,
-                chartType: 'timeseries',
-                // The group-by rows: one per level, plus the lines that carry none.
-                limit: 10,
-            });
-            let estimated = false;
-            const byStart = new Map();
-            for (const point of calc.series ?? []) {
-                const start = point.time ? Date.parse(point.time) : NaN;
-                if (!Number.isFinite(start))
-                    continue;
-                const bucket = byStart.get(start) ??
-                    { start: new Date(start).toISOString(), info: 0, warn: 0, error: 0, unrecorded: 0 };
-                for (const d of point.data ?? []) {
-                    bucket[levelOf(groupValue(d, 'level'))] += countOf(d);
-                    estimated ||= sampled(d);
-                }
-                byStart.set(start, bucket);
-            }
-            const starts = [...byStart.keys()].sort((a, b) => a - b);
-            return {
-                bucketMs: bucketWidth(starts, Math.max(1, Math.round((input.to - input.from) / input.buckets))),
-                buckets: starts
-                    .map((t) => byStart.get(t))
-                    .filter((b) => b.info + b.warn + b.error + b.unrecorded > 0),
-                estimated,
-            };
-        },
-        // #1746: facet counts. One aggregate query per facet, each with every OTHER filter
-        // applied, plus one ungrouped count for the total. In parallel: they are independent
-        // reads of the same window.
-        async tenantRequestFacets(input) {
-            const timeframe = { from: input.from, to: input.to };
-            const keys = input.keys ?? REQUEST_FACET_KEYS;
-            let estimated = false;
-            const [totalCalc, ...perKey] = await Promise.all([
-                queryCalculation(requestFilters(input), timeframe, { chartType: 'aggregate' }),
-                ...keys.map((key) => queryCalculation(requestFilters(input, key), timeframe, {
-                    groupBy: { type: key === 'status' ? 'number' : 'string', value: key },
-                    chartType: 'aggregate',
-                    // Two over the top ten, because the backend applies the limit BEFORE the loop
-                    // below drops the rows that are not values: the group of lines with no value
-                    // for this key (every line older than #1746 has none) and an empty string.
-                    // Either can rank in the top ten and would otherwise take a real value's slot.
-                    limit: REQUEST_FACET_TOP + 2,
-                    // Ordered by the calculation's alias, so the `limit` keeps the most frequent values.
-                    orderBy: { value: 'requests', order: 'desc' },
-                })),
-            ]);
-            const total = (totalCalc.aggregates ?? []).reduce((n, p) => {
-                estimated ||= sampled(p);
-                return n + countOf(p);
-            }, 0);
-            const facets = Object.fromEntries(REQUEST_FACET_KEYS.map((k) => [k, []]));
-            keys.forEach((key, i) => {
-                const values = [];
-                for (const p of perKey[i].aggregates ?? []) {
-                    const value = groupValue(p, key);
-                    // A line with no value for this key (an older line, or a route that is not an
-                    // operation) is not a value anybody can filter on; it is left out of the list,
-                    // and the total above still counts it.
-                    if (typeof value !== 'string' && typeof value !== 'number')
-                        continue;
-                    if (value === '')
-                        continue;
-                    values.push({ value, count: countOf(p) });
-                    estimated ||= sampled(p);
-                }
-                facets[key] = values.sort((a, b) => b.count - a.count).slice(0, REQUEST_FACET_TOP);
-            });
-            return { total, facets, estimated };
-        },
-        // #1747: `ctx.log` lines grouped by template. ONE `calculations` query grouped by
-        // (template, level) answers all three things a pattern shows — its count, its level
-        // split and its histogram — plus one ungrouped count for the total, so a pattern's
-        // share is of every matching line and not only of the templates that made the list.
-        async tenantLogPatterns(input) {
-            const timeframe = { from: input.from, to: input.to };
-            const filters = [
-                { key: 'tenantId', operation: 'eq', type: 'string', value: input.tenantId },
-                { key: 'substrat', operation: 'eq', type: 'string', value: 'log' },
-            ];
-            if (input.scopeId)
-                filters.push({ key: 'scopeId', operation: 'eq', type: 'string', value: input.scopeId });
-            for (const [key, values] of [['level', input.level], ['operation', input.operation]]) {
-                if (!values || values.length === 0)
-                    continue;
-                const leaves = values.map((v) => ({ key, operation: 'eq', type: 'string', value: v }));
-                filters.push(leaves.length === 1 ? leaves[0] : { kind: 'group', filterCombination: 'or', filters: leaves });
-            }
-            // A (template, level) row per group: at most four levels per template, so this many
-            // rows always covers the top templates in full.
-            const rowLimit = LOG_PATTERN_TOP * 4;
-            const [totalCalc, grouped] = await Promise.all([
-                queryCalculation(filters, timeframe, { chartType: 'aggregate' }),
-                queryCalculation(filters, timeframe, {
-                    groupBys: [
-                        { type: 'string', value: 'template' },
-                        { type: 'string', value: 'level' },
-                    ],
-                    granularity: input.buckets,
-                    chartType: 'timeseries_and_aggregate',
-                    limit: rowLimit,
-                    orderBy: { value: 'requests', order: 'desc' },
-                }),
-            ]);
-            let estimated = false;
-            const total = (totalCalc.aggregates ?? []).reduce((n, p) => {
-                estimated ||= sampled(p);
-                return n + countOf(p);
-            }, 0);
-            const levelOfLine = (v) => v === 'debug' || v === 'warn' || v === 'error' ? v : 'info';
-            const byTemplate = new Map();
-            const entry = (template) => {
-                let e = byTemplate.get(template);
-                if (!e) {
-                    e = { levels: { debug: 0, info: 0, warn: 0, error: 0 }, buckets: new Map() };
-                    byTemplate.set(template, e);
-                }
-                return e;
-            };
-            const aggregates = grouped.aggregates ?? [];
-            for (const p of aggregates) {
-                const template = groupValue(p, 'template');
-                // A line with no template is not a `ctx.log` line; it cannot be one pattern.
-                if (typeof template !== 'string' || template === '')
-                    continue;
-                entry(template).levels[levelOfLine(groupValue(p, 'level'))] += countOf(p);
-                estimated ||= sampled(p);
-            }
-            const times = [];
-            for (const point of grouped.series ?? []) {
-                const start = point.time ? Date.parse(point.time) : NaN;
-                if (!Number.isFinite(start))
-                    continue;
-                times.push(start);
-                for (const d of point.data ?? []) {
-                    const template = groupValue(d, 'template');
-                    if (typeof template !== 'string' || !byTemplate.has(template))
-                        continue;
-                    const b = byTemplate.get(template).buckets;
-                    b.set(start, (b.get(start) ?? 0) + countOf(d));
-                }
-            }
-            const patterns = [...byTemplate.entries()]
-                .map(([template, e]) => {
-                const count = e.levels.debug + e.levels.info + e.levels.warn + e.levels.error;
-                const dominant = ['error', 'warn', 'info', 'debug'].reduce((best, l) => e.levels[l] > e.levels[best] ? l : best);
-                return {
-                    template,
-                    count,
-                    share: total > 0 ? count / total : 0,
-                    levels: e.levels,
-                    dominant,
-                    buckets: [...e.buckets.entries()]
-                        .sort(([a], [b]) => a - b)
-                        .map(([t, n]) => ({ start: new Date(t).toISOString(), count: n })),
-                };
-            })
-                .sort((a, b) => b.count - a.count || a.template.localeCompare(b.template))
-                .slice(0, LOG_PATTERN_TOP);
-            return {
-                total,
-                bucketMs: bucketWidth(times, Math.max(1, Math.round((input.to - input.from) / input.buckets))),
-                patterns,
-                // The backend stopped at the row limit, or there were more templates than listed.
-                truncated: aggregates.length >= rowLimit || byTemplate.size > LOG_PATTERN_TOP,
-                estimated,
-            };
-        },
+        // #1746/#1877: the request histogram, facet counts and log patterns — computed once,
+        // from cubes, by `aggregateReads` over the (cached) telemetry source.
+        ...aggregateReads(source),
         // #1746: the requests themselves. The stamped lines alone — no correlation walk,
         // because a request's own line is the whole record and the Requests view has no use
         // for its siblings until a row is opened (which `tenantLogs` with the invocation id
@@ -989,6 +1078,8 @@ export function createCfObservabilityReader(opts) {
         const base = [
             { key: 'tenantId', operation: 'eq', type: 'string', value: input.tenantId },
             { key: 'substrat', operation: 'eq', type: 'string', value: 'invocation' },
+            // #1877: the app's own scripts — the scan boundary and the trust boundary.
+            ...serviceFilter(input.services),
         ];
         if (input.scopeId)
             base.push({ key: 'scopeId', operation: 'eq', type: 'string', value: input.scopeId });
@@ -1052,7 +1143,9 @@ export function createCfObservabilityReader(opts) {
         // because these lines carry both as their own fields. They carry no `vertical`, so a
         // read narrowed by vertical alone (no scope) leaves them out rather than showing a
         // line from another of the tenant's apps.
-        const moduleLines = input.vertical && !input.scopeId ? Promise.resolve([]) : queryRaw(moduleLineFilters(input), timeframe, input.limit);
+        const moduleLines = input.vertical && !input.scopeId
+            ? Promise.resolve([])
+            : queryRaw([...moduleLineFilters(input), ...serviceFilter(input.services)], timeframe, input.limit);
         // A template names `ctx.log` lines only: an invocation line has none.
         if (input.template !== undefined) {
             return (await moduleLines)
@@ -1070,7 +1163,10 @@ export function createCfObservabilityReader(opts) {
                     // in this order, and a busy `= 500` page must not crowd out a newer 502 or
                     // escape just because its query was listed first.
                 ]).then((pages) => pages.flat().sort((a, b) => rawTime(b) - rawTime(a))),
-                queryRaw([{ key: '$metadata.level', operation: 'eq', type: 'string', value: 'error' }], timeframe, phaseOneLimit),
+                queryRaw(
+                // #1877: scoped to the app's scripts too, where known — it was account-wide,
+                // and `ownsInvocation` below still admits nothing without a stamped line.
+                [{ key: '$metadata.level', operation: 'eq', type: 'string', value: 'error' }, ...serviceFilter(input.services)], timeframe, phaseOneLimit),
             ])
             : [await queryRaw(base, timeframe, phaseOneLimit), []];
         const trusted = invocationIds(stamped);
@@ -1090,7 +1186,7 @@ export function createCfObservabilityReader(opts) {
         // `limit` (the route bounds that at 200).
         const linesPerInvocation = input.invocationId !== undefined ? Math.min(input.limit, 200) : MAX_LINES_PER_INVOCATION;
         const sibling = await Promise.all(requestIds.map(async (id) => {
-            const events = await queryRaw([{ key: '$metadata.requestId', operation: 'eq', type: 'string', value: id }], timeframe, linesPerInvocation);
+            const events = await queryRaw([{ key: '$metadata.requestId', operation: 'eq', type: 'string', value: id }, ...serviceFilter(input.services)], timeframe, linesPerInvocation);
             // An account-wide candidate earns its place only by producing this tenant's
             // stamped line. No stamped line, or somebody else's, and the whole invocation is
             // dropped — a line shown to the wrong tenant is exactly the leak the tenant grain
@@ -1114,10 +1210,13 @@ export function createCfObservabilityReader(opts) {
             const id = typeof metadata['id'] === 'string' ? metadata['id'] : JSON.stringify(e);
             byId.set(id, e);
         }
+        // ONE row per request: the runtime's own invocation record is folded into the stamped
+        // line that describes the same request, which says everything it does and more.
+        const runtimeOf = foldRuntimeRecords([...byId.values()]);
         const level = input.level?.toLowerCase();
         const search = input.search;
-        return [...byId.values()]
-            .map((e) => projectEvent(e))
+        return runtimeOf.kept
+            .map((e) => withRuntimeTimings(projectEvent(e), runtimeOf.byRequest))
             .map((e) => describeModuleLine(describeInvocation(e)))
             // A plain comparison is enough only because `describeInvocation` ran first: a stamped
             // line has no level of its own (Cloudflare sets `$metadata.level` for a string log
@@ -1161,6 +1260,7 @@ export function createCfObservabilityReader(opts) {
             timeframe,
             chartType: shape.chartType,
             ...(shape.granularity === undefined ? {} : { granularity: shape.granularity }),
+            ...(shape.offsetBy === undefined ? {} : { offsetBy: shape.offsetBy }),
             parameters: {
                 datasets: ['cloudflare-workers'],
                 filters,
@@ -1170,11 +1270,12 @@ export function createCfObservabilityReader(opts) {
                 ...(shape.orderBy ? { orderBy: shape.orderBy } : {}),
             },
         });
-        const json = (await res.json());
-        if (!res.ok || json.success === false) {
-            const message = json.errors?.map((e) => e.message).join('; ') || `HTTP ${res.status}`;
-            throw new Error(`Cloudflare telemetry query failed: ${message}`);
-        }
+        const text = await res.text();
+        if (!res.ok)
+            throw telemetryFailure(res.status, text);
+        const json = JSON.parse(text);
+        if (json.success === false)
+            throw telemetryFailure(502, text);
         // One calculation was asked for, so the first answer is it. An answer with none is an
         // empty window, not an error: the backend omits a calculation that matched nothing.
         return json.result?.calculations?.[0] ?? {};
@@ -1196,11 +1297,12 @@ export function createCfObservabilityReader(opts) {
             parameters: { datasets: ['cloudflare-workers'], filters, limit },
             limit,
         });
-        const json = (await res.json());
-        if (!res.ok || json.success === false) {
-            const message = json.errors?.map((e) => e.message).join('; ') || `HTTP ${res.status}`;
-            throw new Error(`Cloudflare telemetry query failed: ${message}`);
-        }
+        const text = await res.text();
+        if (!res.ok)
+            throw telemetryFailure(res.status, text);
+        const json = JSON.parse(text);
+        if (json.success === false)
+            throw telemetryFailure(502, text);
         const outer = json.result?.events;
         return (Array.isArray(outer) ? outer : (outer?.events ?? []));
     }