@decocms/blocks 7.81.2 → 7.81.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/sdk/otel.ts +8 -3
- package/src/sdk/otelHttpMeter.test.ts +45 -5
- package/src/sdk/otelHttpMeter.ts +36 -21
- package/src/sdk/otelHttpTracer.ts +2 -3
package/package.json
CHANGED
package/src/sdk/otel.ts
CHANGED
|
@@ -928,8 +928,8 @@ export function bootObservability(opts: OtelOptions, env: Record<string, unknown
|
|
|
928
928
|
// via the per-site dataset.
|
|
929
929
|
// - OTLP/HTTP (env `DECO_OTEL_METRICS_ENDPOINT`): SRE-grade rollups
|
|
930
930
|
// landed in ClickHouse `otel_metrics_{sum,gauge,histogram}` via the
|
|
931
|
-
// `deco-otel-ingest` Worker.
|
|
932
|
-
// buffer
|
|
931
|
+
// `deco-otel-ingest` Worker. Delta temporality, per-isolate
|
|
932
|
+
// buffer drained on each flush by `ctx.waitUntil` in `instrumentWorker`.
|
|
933
933
|
//
|
|
934
934
|
// The two emitters are NOT redundant — AE keeps per-request per-path
|
|
935
935
|
// dimensions; OTLP carries the same metric names at coarser cardinality
|
|
@@ -945,9 +945,14 @@ export function bootObservability(opts: OtelOptions, env: Record<string, unknown
|
|
|
945
945
|
const otlpEndpoint = (env[otlpEnvVar] as string | undefined) ?? "";
|
|
946
946
|
const otlpEnabled = opts.otlpMetricsEnabled !== false && otlpEndpoint.length > 0;
|
|
947
947
|
if (otlpEnabled) {
|
|
948
|
+
// Metrics drop `service.instance.id`: delta points from different
|
|
949
|
+
// isolates are meant to be summed, and a per-isolate UUID would split
|
|
950
|
+
// them back into one series per isolate (decocms/blocks#750). Logs and
|
|
951
|
+
// traces keep it for per-instance attribution.
|
|
952
|
+
const { [ATTR_SERVICE_INSTANCE_ID]: _instanceId, ...metricsResource } = floor;
|
|
948
953
|
state.otlpMeter = createOtlpHttpMeterAdapter({
|
|
949
954
|
endpoint: otlpEndpoint,
|
|
950
|
-
resourceAttributes:
|
|
955
|
+
resourceAttributes: metricsResource,
|
|
951
956
|
scopeVersion: decoRuntimeVersion,
|
|
952
957
|
headers: otlpHeaders,
|
|
953
958
|
fetchImpl: opts.otlpMetricsFetchImpl,
|
|
@@ -97,7 +97,7 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
97
97
|
vi.restoreAllMocks();
|
|
98
98
|
});
|
|
99
99
|
|
|
100
|
-
it("counterInc accumulates per attr-key and exports
|
|
100
|
+
it("counterInc accumulates per attr-key and exports delta sum", async () => {
|
|
101
101
|
const { impl, calls } = captureFetch();
|
|
102
102
|
const meter = buildAdapter({ fetchImpl: impl });
|
|
103
103
|
|
|
@@ -113,7 +113,7 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
113
113
|
const m = payload.resourceMetrics[0].scopeMetrics[0].metrics[0];
|
|
114
114
|
expect(m.name).toBe("deco.http.requests");
|
|
115
115
|
expect(m.sum?.isMonotonic).toBe(true);
|
|
116
|
-
expect(m.sum?.aggregationTemporality).toBe(
|
|
116
|
+
expect(m.sum?.aggregationTemporality).toBe(1);
|
|
117
117
|
expect(m.sum?.dataPoints).toHaveLength(2);
|
|
118
118
|
|
|
119
119
|
// Sort for stable assertion order.
|
|
@@ -124,6 +124,43 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
124
124
|
expect(points[1].asDouble).toBe(1); // POST+5xx
|
|
125
125
|
});
|
|
126
126
|
|
|
127
|
+
it("kind-mismatch is still caught after the buffer was flushed", async () => {
|
|
128
|
+
const onError = vi.fn();
|
|
129
|
+
const { impl, calls } = captureFetch();
|
|
130
|
+
const meter = buildAdapter({ fetchImpl: impl, onError });
|
|
131
|
+
|
|
132
|
+
meter.counterInc("x", 1);
|
|
133
|
+
await meter.flush(true);
|
|
134
|
+
meter.histogramRecord("x", 5);
|
|
135
|
+
|
|
136
|
+
expect(onError).toHaveBeenCalledWith("kind-mismatch", expect.any(Error));
|
|
137
|
+
expect(meter.pendingDatapointCount()).toBe(0);
|
|
138
|
+
expect(calls).toHaveLength(1);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("each flush carries only its own window; idle series are not re-sent", async () => {
|
|
142
|
+
const { impl, calls } = captureFetch();
|
|
143
|
+
let mockNow = 1_000;
|
|
144
|
+
const meter = buildAdapter({ fetchImpl: impl, nowMs: () => mockNow });
|
|
145
|
+
|
|
146
|
+
meter.histogramRecord("lat", 10, { route: "/a" });
|
|
147
|
+
meter.histogramRecord("lat", 20, { route: "/b" });
|
|
148
|
+
await meter.flush(true);
|
|
149
|
+
|
|
150
|
+
mockNow = 7_000;
|
|
151
|
+
meter.histogramRecord("lat", 30, { route: "/a" });
|
|
152
|
+
await meter.flush(true);
|
|
153
|
+
|
|
154
|
+
expect(calls).toHaveLength(2);
|
|
155
|
+
const second = JSON.parse(calls[1].init!.body as string) as OtlpPayload;
|
|
156
|
+
const h = second.resourceMetrics[0].scopeMetrics[0].metrics[0].histogram!;
|
|
157
|
+
expect(h.aggregationTemporality).toBe(1);
|
|
158
|
+
expect(h.dataPoints).toHaveLength(1); // `/b` was idle — not re-exported
|
|
159
|
+
expect(h.dataPoints[0].count).toBe("1");
|
|
160
|
+
expect(h.dataPoints[0].startTimeUnixNano).toBe("1000000000"); // previous flush
|
|
161
|
+
expect(h.dataPoints[0].timeUnixNano).toBe("7000000000");
|
|
162
|
+
});
|
|
163
|
+
|
|
127
164
|
it("histogramRecord assigns buckets correctly and reports count/sum/min/max", async () => {
|
|
128
165
|
const { impl, calls } = captureFetch();
|
|
129
166
|
const meter = buildAdapter({
|
|
@@ -152,7 +189,7 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
152
189
|
|
|
153
190
|
const payload = JSON.parse(calls[0].init!.body as string) as OtlpPayload;
|
|
154
191
|
const m = payload.resourceMetrics[0].scopeMetrics[0].metrics[0];
|
|
155
|
-
expect(m.histogram?.aggregationTemporality).toBe(
|
|
192
|
+
expect(m.histogram?.aggregationTemporality).toBe(1);
|
|
156
193
|
expect(m.histogram?.dataPoints).toHaveLength(1);
|
|
157
194
|
const dp = m.histogram!.dataPoints[0];
|
|
158
195
|
expect(dp.count).toBe("12");
|
|
@@ -210,8 +247,8 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
210
247
|
|
|
211
248
|
it("non-200 HTTP response surfaces via onError but does not throw", async () => {
|
|
212
249
|
const onError = vi.fn();
|
|
213
|
-
const non200: typeof fetch = vi.fn(
|
|
214
|
-
new Response("oops", { status: 500 }),
|
|
250
|
+
const non200: typeof fetch = vi.fn(
|
|
251
|
+
async () => new Response("oops", { status: 500 }),
|
|
215
252
|
) as unknown as typeof fetch;
|
|
216
253
|
const meter = buildAdapter({ fetchImpl: non200, onError });
|
|
217
254
|
meter.counterInc("x", 1);
|
|
@@ -244,6 +281,8 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
244
281
|
meter.counterInc("a", 1, { k: "1" });
|
|
245
282
|
await meter.flush();
|
|
246
283
|
expect(calls).toHaveLength(1);
|
|
284
|
+
// Delta: the flushed window is gone from the buffer.
|
|
285
|
+
expect(meter.pendingDatapointCount()).toBe(0);
|
|
247
286
|
|
|
248
287
|
// 2nd flush at t=2s — cooldown not elapsed AND buffer below cap → no-op
|
|
249
288
|
mockNow += 2000;
|
|
@@ -254,6 +293,7 @@ describe("createOtlpHttpMeterAdapter — buffer + flush", () => {
|
|
|
254
293
|
// 3rd flush at t=2.5s — still under cooldown but buffer at cap (3) → flush
|
|
255
294
|
mockNow += 500;
|
|
256
295
|
meter.counterInc("a", 1, { k: "3" });
|
|
296
|
+
meter.counterInc("a", 1, { k: "4" });
|
|
257
297
|
expect(meter.pendingDatapointCount()).toBe(3);
|
|
258
298
|
await meter.flush();
|
|
259
299
|
expect(calls).toHaveLength(2);
|
package/src/sdk/otelHttpMeter.ts
CHANGED
|
@@ -9,10 +9,15 @@
|
|
|
9
9
|
* ingestor's per-POST charge is captured in the cost model in
|
|
10
10
|
* `docs/observability.md`.
|
|
11
11
|
*
|
|
12
|
-
* **Aggregation model.** Buffers are per-isolate
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* **Aggregation model.** Buffers are per-isolate and exported with
|
|
13
|
+
* `AggregationTemporality = DELTA`: every flush sends only what was recorded
|
|
14
|
+
* since the previous flush, then starts an empty window. Cumulative export
|
|
15
|
+
* re-sent every series an isolate had ever seen on every flush — one
|
|
16
|
+
* histogram row per series per 5 s for the isolate's whole life — and forced
|
|
17
|
+
* a per-isolate `service.instance.id` on the resource to keep the running
|
|
18
|
+
* totals apart, which made every isolate its own set of series. Measured in
|
|
19
|
+
* stats-lake: ~600M rows/day in `otel_metrics_histogram` (decocms/blocks#750).
|
|
20
|
+
* Delta points from different isolates sum correctly without an instance id.
|
|
16
21
|
*
|
|
17
22
|
* **Flush triggers.**
|
|
18
23
|
* 1. `flush()` — caller-driven, used by `workerEntry` inside
|
|
@@ -177,10 +182,14 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
177
182
|
const onError = options.onError;
|
|
178
183
|
const metricMetadata = options.metricMetadata ?? {};
|
|
179
184
|
|
|
180
|
-
// Buffer state — per-isolate,
|
|
185
|
+
// Buffer state — per-isolate, swapped out on every flush (DELTA temporality).
|
|
181
186
|
const metrics = new Map<string, MetricEntry>();
|
|
182
|
-
//
|
|
183
|
-
|
|
187
|
+
// Kind each name was first recorded as. Outlives the per-flush reset so a
|
|
188
|
+
// name can't come back as a different kind in a later window.
|
|
189
|
+
const kinds = new Map<string, MetricKind>();
|
|
190
|
+
// Start of the current delta window — counters/histograms report it as
|
|
191
|
+
// their startTime. Advances when a flush snapshots the buffer.
|
|
192
|
+
let windowStartMs = now();
|
|
184
193
|
// Per-isolate flush throttle.
|
|
185
194
|
let lastFlushAt = 0;
|
|
186
195
|
let inflight: Promise<void> | null = null;
|
|
@@ -209,16 +218,13 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
209
218
|
name: string,
|
|
210
219
|
kind: MetricKind,
|
|
211
220
|
): { entry: MetricEntry | null; isNewName: boolean } {
|
|
212
|
-
const
|
|
213
|
-
if (
|
|
214
|
-
|
|
215
|
-
onError?.(
|
|
216
|
-
"kind-mismatch",
|
|
217
|
-
new Error(`metric "${name}" already registered as ${existing.kind}`),
|
|
218
|
-
);
|
|
221
|
+
const registered = kinds.get(name);
|
|
222
|
+
if (registered && registered !== kind) {
|
|
223
|
+
onError?.("kind-mismatch", new Error(`metric "${name}" already registered as ${registered}`));
|
|
219
224
|
return { entry: null, isNewName: false };
|
|
220
225
|
}
|
|
221
|
-
|
|
226
|
+
const existing = metrics.get(name);
|
|
227
|
+
return existing ? { entry: existing, isNewName: false } : { entry: null, isNewName: true };
|
|
222
228
|
}
|
|
223
229
|
|
|
224
230
|
function materializeEntry(name: string, kind: MetricKind): MetricEntry {
|
|
@@ -227,6 +233,7 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
227
233
|
else if (kind === "gauge") entry.gauge = new Map();
|
|
228
234
|
else entry.histogram = new Map();
|
|
229
235
|
metrics.set(name, entry);
|
|
236
|
+
kinds.set(name, kind);
|
|
230
237
|
return entry;
|
|
231
238
|
}
|
|
232
239
|
|
|
@@ -256,7 +263,7 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
256
263
|
point = {
|
|
257
264
|
value: 0,
|
|
258
265
|
attrs: labels ? { ...labels } : {},
|
|
259
|
-
startTimeUnixNano: msToNs(
|
|
266
|
+
startTimeUnixNano: msToNs(windowStartMs),
|
|
260
267
|
};
|
|
261
268
|
entry.counter.set(key, point);
|
|
262
269
|
}
|
|
@@ -303,7 +310,7 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
303
310
|
max: Number.NEGATIVE_INFINITY,
|
|
304
311
|
bucketCounts: new Array(bounds.length + 1).fill(0),
|
|
305
312
|
attrs: labels ? { ...labels } : {},
|
|
306
|
-
startTimeUnixNano: msToNs(
|
|
313
|
+
startTimeUnixNano: msToNs(windowStartMs),
|
|
307
314
|
};
|
|
308
315
|
entry.histogram.set(key, point);
|
|
309
316
|
}
|
|
@@ -325,8 +332,16 @@ export function createOtlpHttpMeterAdapter(options: OtlpHttpMeterOptions): OtlpH
|
|
|
325
332
|
async function doFlush(): Promise<void> {
|
|
326
333
|
if (metrics.size === 0) return;
|
|
327
334
|
|
|
328
|
-
|
|
329
|
-
|
|
335
|
+
// Snapshot and reset synchronously, before the first await, so points
|
|
336
|
+
// recorded while the POST is in flight land in the next window.
|
|
337
|
+
// ponytail: a failed POST drops its window (≤ minFlushIntervalMs of data)
|
|
338
|
+
// instead of merging it back; add a retry buffer if loss shows up.
|
|
339
|
+
const snapshot = new Map(metrics);
|
|
340
|
+
metrics.clear();
|
|
341
|
+
const flushAt = now();
|
|
342
|
+
windowStartMs = flushAt;
|
|
343
|
+
const flushAtNs = msToNs(flushAt);
|
|
344
|
+
const payload = serializeOtlp(snapshot, {
|
|
330
345
|
resourceAttributes,
|
|
331
346
|
scopeName,
|
|
332
347
|
scopeVersion,
|
|
@@ -459,7 +474,7 @@ function serializeOtlp(
|
|
|
459
474
|
description,
|
|
460
475
|
unit,
|
|
461
476
|
sum: {
|
|
462
|
-
aggregationTemporality:
|
|
477
|
+
aggregationTemporality: 1, // DELTA
|
|
463
478
|
isMonotonic: true,
|
|
464
479
|
dataPoints,
|
|
465
480
|
},
|
|
@@ -501,7 +516,7 @@ function serializeOtlp(
|
|
|
501
516
|
description,
|
|
502
517
|
unit,
|
|
503
518
|
histogram: {
|
|
504
|
-
aggregationTemporality:
|
|
519
|
+
aggregationTemporality: 1, // DELTA
|
|
505
520
|
dataPoints,
|
|
506
521
|
},
|
|
507
522
|
});
|
|
@@ -21,9 +21,8 @@
|
|
|
21
21
|
* exported regardless of the rate.
|
|
22
22
|
*
|
|
23
23
|
* **Buffer.** Spans are buffered per-isolate until `flush()` ships them.
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* window. Cap (`maxBufferSpans`) protects against runaway trace volume.
|
|
24
|
+
* Like metrics (DELTA temporality), the buffer drains on flush and starts
|
|
25
|
+
* empty for the next window; each span is exported once. Cap (`maxBufferSpans`) protects against runaway trace volume.
|
|
27
26
|
*
|
|
28
27
|
* **Parent linkage.** `startSpan` reads the active span from a caller-
|
|
29
28
|
* supplied accessor (`getActiveSpanForParent`) — the framework wires this
|