@oxygen-agent/cli 1.800.1 → 1.804.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/node_modules/@oxygen/shared/dist/error-redaction.d.ts +17 -0
- package/node_modules/@oxygen/shared/dist/error-redaction.js +28 -0
- package/node_modules/@oxygen/shared/dist/index.d.ts +1 -0
- package/node_modules/@oxygen/shared/dist/index.js +1 -0
- package/node_modules/@oxygen/shared/dist/log-collapse.d.ts +141 -0
- package/node_modules/@oxygen/shared/dist/log-collapse.js +242 -0
- package/node_modules/@oxygen/shared/dist/log.d.ts +1 -0
- package/node_modules/@oxygen/shared/dist/log.js +49 -1
- package/node_modules/@oxygen/shared/dist/version.d.ts +1 -1
- package/node_modules/@oxygen/shared/dist/version.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -111,3 +111,20 @@ export type ErrorCauseEntry = {
|
|
|
111
111
|
};
|
|
112
112
|
export declare function errorCauseChain(error: unknown, maxDepth?: number): ErrorCauseEntry[];
|
|
113
113
|
export declare function describeErrorWithCauses(error: unknown, maxDepth?: number): string;
|
|
114
|
+
/**
|
|
115
|
+
* The operator-side view of a provider's response body, for the `log()` line
|
|
116
|
+
* that accompanies a collapsed provider failure (ADR 0023 §2).
|
|
117
|
+
*
|
|
118
|
+
* Keeps the description fields a human reads to work out what the vendor
|
|
119
|
+
* objected to, drops everything else — echoed request payloads, pagination
|
|
120
|
+
* blobs, nested account structures. `log()`'s own field sanitizer already
|
|
121
|
+
* redacts secret-NAMED keys, so this is not a credential guard; it is what keeps
|
|
122
|
+
* a withheld-detail line the size of a sentence instead of a page.
|
|
123
|
+
*
|
|
124
|
+
* Lifted out of `cli-http.ts` (its only caller until now) because the
|
|
125
|
+
* observability projection collapses the SAME provider errors on the write path
|
|
126
|
+
* and logged `provider.body` raw. Two log shapes for one fault made an Axiom
|
|
127
|
+
* query over withheld provider detail depend on which surface happened to catch
|
|
128
|
+
* it; there is one shape now.
|
|
129
|
+
*/
|
|
130
|
+
export declare function summarizeProviderBody(body: unknown): unknown;
|
|
@@ -275,6 +275,34 @@ function describeCauseEntry(value) {
|
|
|
275
275
|
}
|
|
276
276
|
return { name: typeof value, code: null, message: truncate(String(value), MAX_CAUSE_MESSAGE_LENGTH) };
|
|
277
277
|
}
|
|
278
|
+
/**
|
|
279
|
+
* The operator-side view of a provider's response body, for the `log()` line
|
|
280
|
+
* that accompanies a collapsed provider failure (ADR 0023 §2).
|
|
281
|
+
*
|
|
282
|
+
* Keeps the description fields a human reads to work out what the vendor
|
|
283
|
+
* objected to, drops everything else — echoed request payloads, pagination
|
|
284
|
+
* blobs, nested account structures. `log()`'s own field sanitizer already
|
|
285
|
+
* redacts secret-NAMED keys, so this is not a credential guard; it is what keeps
|
|
286
|
+
* a withheld-detail line the size of a sentence instead of a page.
|
|
287
|
+
*
|
|
288
|
+
* Lifted out of `cli-http.ts` (its only caller until now) because the
|
|
289
|
+
* observability projection collapses the SAME provider errors on the write path
|
|
290
|
+
* and logged `provider.body` raw. Two log shapes for one fault made an Axiom
|
|
291
|
+
* query over withheld provider detail depend on which surface happened to catch
|
|
292
|
+
* it; there is one shape now.
|
|
293
|
+
*/
|
|
294
|
+
export function summarizeProviderBody(body) {
|
|
295
|
+
if (!isRecord(body))
|
|
296
|
+
return body;
|
|
297
|
+
const summary = {};
|
|
298
|
+
for (const key of ["code", "message", "error", "error_code", "filter_error", "type", "detail", "details", "text"]) {
|
|
299
|
+
const value = body[key];
|
|
300
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
301
|
+
summary[key] = value;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return Object.keys(summary).length > 0 ? summary : null;
|
|
305
|
+
}
|
|
278
306
|
function truncate(value, maxLength) {
|
|
279
307
|
return value.length > maxLength ? `${value.slice(0, maxLength - 3)}...` : value;
|
|
280
308
|
}
|
|
@@ -59,6 +59,7 @@ export * from "./dnc-identities.js";
|
|
|
59
59
|
export * from "./table-limits.js";
|
|
60
60
|
export * from "./table-capacity.js";
|
|
61
61
|
export * from "./log.js";
|
|
62
|
+
export * from "./log-collapse.js";
|
|
62
63
|
export * from "./axiom-field-budget.js";
|
|
63
64
|
export { redactSecretsInString, sanitizeLogFields } from "./redaction.js";
|
|
64
65
|
export * from "./provider-request-outcomes.js";
|
|
@@ -59,6 +59,7 @@ export * from "./dnc-identities.js";
|
|
|
59
59
|
export * from "./table-limits.js";
|
|
60
60
|
export * from "./table-capacity.js";
|
|
61
61
|
export * from "./log.js";
|
|
62
|
+
export * from "./log-collapse.js";
|
|
62
63
|
export * from "./axiom-field-budget.js";
|
|
63
64
|
// Narrow, deliberate export (ADR 0014): lets telemetry emitters regression-test
|
|
64
65
|
// their field names against the REAL log sanitizer — the unanchored
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Collapse a repeating log line to a decaying schedule WITHOUT losing the volume
|
|
3
|
+
* it absorbs.
|
|
4
|
+
*
|
|
5
|
+
* The shape is lifted from the BYOK would-block dedupe in
|
|
6
|
+
* `packages/providers/src/provider-fetch.ts`, where one organization on one
|
|
7
|
+
* provider produced 20,404 identical `provider.byok_daily_cap_would_block` lines
|
|
8
|
+
* in 48h — the single largest log-volume contributor in the 2026-08 sweep, and
|
|
9
|
+
* enough on its own to starve the Axiom shipper. Every hot path that logs a
|
|
10
|
+
* *condition* rather than an *event* has that failure mode, so the state machine
|
|
11
|
+
* lives here once instead of being re-derived per caller.
|
|
12
|
+
*
|
|
13
|
+
* What it is NOT: a sampler. A sampled line cannot answer "how many did this key
|
|
14
|
+
* really have", and the whole reason a flood is worth logging at all is the count.
|
|
15
|
+
* Two rules keep the accounting lossless:
|
|
16
|
+
*
|
|
17
|
+
* - every emitted line carries the window's running `occurrences` plus the
|
|
18
|
+
* `suppressed` count it absorbed since the previous line, and
|
|
19
|
+
* - any window whose collapsed tail would otherwise die with its state — the
|
|
20
|
+
* window rolled, or the entry was evicted — hands back a closing rollup so the
|
|
21
|
+
* caller can emit one final line for it.
|
|
22
|
+
*
|
|
23
|
+
* KNOWN LIMITATION — there is no flush timer. A key stops being observed the
|
|
24
|
+
* moment its traffic stops, so its last collapsed tail is held until that key's
|
|
25
|
+
* next occurrence, a window roll, or an eviction retires it — and is lost
|
|
26
|
+
* entirely if the process exits first (Vercel functions freeze, Fly machines are
|
|
27
|
+
* replaced on deploy). That bounds the unreported remainder to one key's
|
|
28
|
+
* sub-schedule tail rather than a whole window, which is the trade this makes
|
|
29
|
+
* deliberately: a timer would keep a process-wide handle alive on serverless and
|
|
30
|
+
* would fire in a context with no request/log context to emit into. Callers that
|
|
31
|
+
* need the tail promptly should choose a shorter final delay, not add a timer
|
|
32
|
+
* here.
|
|
33
|
+
*
|
|
34
|
+
* FIELD SHAPE: the counts this returns are meant to ride in the `worker_fields`
|
|
35
|
+
* overflow map (they are deliberately NOT in AXIOM_STABLE_FIELDS) — `oxygen-logs`
|
|
36
|
+
* is at its column cap, and an allowlisted name with no column rejects the whole
|
|
37
|
+
* ingest batch. Query them as `['worker_fields']['occurrences']`, never flat.
|
|
38
|
+
*
|
|
39
|
+
* NOT a substitute for the log line: an OTel counter is not a backstop for
|
|
40
|
+
* collapsed volume. `registerOTel()` on web is called with no metric readers and
|
|
41
|
+
* the worker only builds one when AXIOM_METRICS_DATASET is set (it is not), so
|
|
42
|
+
* every recordTelemetryCounter call in this repo is a no-op today. If a signal
|
|
43
|
+
* matters it stays in the LOG — which is what `occurrences`/`suppressed` are for.
|
|
44
|
+
*/
|
|
45
|
+
/** Why a window's state was retired — carried on the closing rollup line. */
|
|
46
|
+
export type LogCollapseRollupReason = "window_rolled" | "state_evicted";
|
|
47
|
+
/** Counts for one live line, or null when this occurrence is collapsed. */
|
|
48
|
+
export type LogCollapseEmit = {
|
|
49
|
+
/** Occurrences for this key in this window, emitted and suppressed alike. */
|
|
50
|
+
occurrences: number;
|
|
51
|
+
/** Occurrences collapsed into this line since the previous emitted one. */
|
|
52
|
+
suppressed: number;
|
|
53
|
+
/** Distinct labels behind the counts, sorted, capped at `maxLabels`. */
|
|
54
|
+
labels: string[];
|
|
55
|
+
/** True once an unseen label was dropped by that cap. */
|
|
56
|
+
labelsTruncated: boolean;
|
|
57
|
+
/** Window this line belongs to, as epoch ms — floor(now / windowMs) * windowMs. */
|
|
58
|
+
windowStartMs: number;
|
|
59
|
+
};
|
|
60
|
+
/** Closing totals for a window that will emit no further live line. */
|
|
61
|
+
export type LogCollapseRollup = LogCollapseEmit & {
|
|
62
|
+
/** The `record()` key, so the caller can reconstruct its own dimensions. */
|
|
63
|
+
key: string;
|
|
64
|
+
/** Lines emitted for this key in this window before it was retired. */
|
|
65
|
+
emitted: number;
|
|
66
|
+
reason: LogCollapseRollupReason;
|
|
67
|
+
};
|
|
68
|
+
export type LogCollapseDecision = {
|
|
69
|
+
emit: LogCollapseEmit | null;
|
|
70
|
+
/** Windows retired by this call. Always log them — suppression must not vanish. */
|
|
71
|
+
rollups: LogCollapseRollup[];
|
|
72
|
+
};
|
|
73
|
+
export type LogCollapserOptions = {
|
|
74
|
+
/**
|
|
75
|
+
* Length of the counting window in ms. A key's line budget resets when the
|
|
76
|
+
* window rolls, and window boundaries are absolute (`floor(now / windowMs)`),
|
|
77
|
+
* so a caller whose underlying bucket uses the same `startOfWindow` arithmetic
|
|
78
|
+
* — e.g. a rate-limit window — rolls over together with it.
|
|
79
|
+
*/
|
|
80
|
+
windowMs: number;
|
|
81
|
+
/**
|
|
82
|
+
* Delay from an emitted line to the next one the same key may emit, indexed by
|
|
83
|
+
* lines already emitted in this window. The LAST entry repeats forever, so
|
|
84
|
+
* `[5min, 30min, 1h]` means: immediately, +5min, +30min, then hourly. Front-load
|
|
85
|
+
* it so a newly-firing key is visible at once and again while a human is still
|
|
86
|
+
* looking, and decay it so a key that stays pinned all day costs tens of lines
|
|
87
|
+
* per process instead of tens of thousands. An empty schedule degrades to one
|
|
88
|
+
* line per key per window.
|
|
89
|
+
*/
|
|
90
|
+
delaysMs: readonly number[];
|
|
91
|
+
/**
|
|
92
|
+
* Bound on live keys, so a pathological fan-out (many orgs x many providers)
|
|
93
|
+
* cannot grow the map without limit inside a long-lived worker process.
|
|
94
|
+
*/
|
|
95
|
+
maxEntries: number;
|
|
96
|
+
/**
|
|
97
|
+
* Distinct labels tracked per key. Bounded because a label space (operations,
|
|
98
|
+
* error codes, routes) is open-ended, and this exists only to make the bias of
|
|
99
|
+
* any single-sample label field on the emitted line explicit.
|
|
100
|
+
*/
|
|
101
|
+
maxLabels: number;
|
|
102
|
+
};
|
|
103
|
+
export type LogCollapser = {
|
|
104
|
+
/**
|
|
105
|
+
* Record one occurrence for `key` and decide what may be logged. `emit` carries
|
|
106
|
+
* the counts for a live line or is null when this occurrence is collapsed into a
|
|
107
|
+
* later one; `rollups` carries the closing totals of any window this call
|
|
108
|
+
* retired. The occurrence count is ALWAYS incremented, so a collapsed occurrence
|
|
109
|
+
* still surfaces in the next emitted line or in that window's rollup.
|
|
110
|
+
*
|
|
111
|
+
* Choose the key at the granularity of the CONDITION, not of the call: keying
|
|
112
|
+
* more finely than the thing being reported re-opens the flood for any caller
|
|
113
|
+
* that spreads its calls across the finer dimension. Encode whatever the caller
|
|
114
|
+
* must name on the rollup into the key with a separator its parts cannot contain
|
|
115
|
+
* (a NUL is the usual choice) — rollups hand the key back verbatim.
|
|
116
|
+
*/
|
|
117
|
+
record: (key: string, label?: string) => LogCollapseDecision;
|
|
118
|
+
/** Test-only: lets a suite re-observe the leading-edge line of a window. */
|
|
119
|
+
reset: () => void;
|
|
120
|
+
};
|
|
121
|
+
export declare function createLogCollapser(options: LogCollapserOptions): LogCollapser;
|
|
122
|
+
/**
|
|
123
|
+
* Claim the single log slot for `key` in the current window: true the first time,
|
|
124
|
+
* false for every repeat until the window rolls.
|
|
125
|
+
*
|
|
126
|
+
* The cheaper sibling of `createLogCollapser` and the right tool when the line
|
|
127
|
+
* reports a STATE TRANSITION rather than a rate — a budget threshold crossed, a
|
|
128
|
+
* connector going unhealthy, a floor being hit. Those lines answer "is this
|
|
129
|
+
* workspace in that state today", so a count adds nothing and a per-call emission
|
|
130
|
+
* is pure flood (the shape that has blinded this dataset before). When the count
|
|
131
|
+
* IS the point, use the collapser, which keeps `occurrences`/`suppressed`.
|
|
132
|
+
*
|
|
133
|
+
* Modelled on `claimThresholdLogSlot`
|
|
134
|
+
* (packages/integrations/src/budget-policy.ts), including its accepted trade: at
|
|
135
|
+
* the bound the set is cleared wholesale rather than evicted one by one, so a key
|
|
136
|
+
* that was already claimed can emit a second line in the same window. Bounded
|
|
137
|
+
* memory beats exactly-once for a line that is idempotent to re-read.
|
|
138
|
+
*/
|
|
139
|
+
export declare function claimOncePerWindow(key: string, windowMs?: number): boolean;
|
|
140
|
+
/** Test-only: clear the per-process once-per-window claims between cases. */
|
|
141
|
+
export declare function resetOncePerWindowClaims(): void;
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Collapse a repeating log line to a decaying schedule WITHOUT losing the volume
|
|
3
|
+
* it absorbs.
|
|
4
|
+
*
|
|
5
|
+
* The shape is lifted from the BYOK would-block dedupe in
|
|
6
|
+
* `packages/providers/src/provider-fetch.ts`, where one organization on one
|
|
7
|
+
* provider produced 20,404 identical `provider.byok_daily_cap_would_block` lines
|
|
8
|
+
* in 48h — the single largest log-volume contributor in the 2026-08 sweep, and
|
|
9
|
+
* enough on its own to starve the Axiom shipper. Every hot path that logs a
|
|
10
|
+
* *condition* rather than an *event* has that failure mode, so the state machine
|
|
11
|
+
* lives here once instead of being re-derived per caller.
|
|
12
|
+
*
|
|
13
|
+
* What it is NOT: a sampler. A sampled line cannot answer "how many did this key
|
|
14
|
+
* really have", and the whole reason a flood is worth logging at all is the count.
|
|
15
|
+
* Two rules keep the accounting lossless:
|
|
16
|
+
*
|
|
17
|
+
* - every emitted line carries the window's running `occurrences` plus the
|
|
18
|
+
* `suppressed` count it absorbed since the previous line, and
|
|
19
|
+
* - any window whose collapsed tail would otherwise die with its state — the
|
|
20
|
+
* window rolled, or the entry was evicted — hands back a closing rollup so the
|
|
21
|
+
* caller can emit one final line for it.
|
|
22
|
+
*
|
|
23
|
+
* KNOWN LIMITATION — there is no flush timer. A key stops being observed the
|
|
24
|
+
* moment its traffic stops, so its last collapsed tail is held until that key's
|
|
25
|
+
* next occurrence, a window roll, or an eviction retires it — and is lost
|
|
26
|
+
* entirely if the process exits first (Vercel functions freeze, Fly machines are
|
|
27
|
+
* replaced on deploy). That bounds the unreported remainder to one key's
|
|
28
|
+
* sub-schedule tail rather than a whole window, which is the trade this makes
|
|
29
|
+
* deliberately: a timer would keep a process-wide handle alive on serverless and
|
|
30
|
+
* would fire in a context with no request/log context to emit into. Callers that
|
|
31
|
+
* need the tail promptly should choose a shorter final delay, not add a timer
|
|
32
|
+
* here.
|
|
33
|
+
*
|
|
34
|
+
* FIELD SHAPE: the counts this returns are meant to ride in the `worker_fields`
|
|
35
|
+
* overflow map (they are deliberately NOT in AXIOM_STABLE_FIELDS) — `oxygen-logs`
|
|
36
|
+
* is at its column cap, and an allowlisted name with no column rejects the whole
|
|
37
|
+
* ingest batch. Query them as `['worker_fields']['occurrences']`, never flat.
|
|
38
|
+
*
|
|
39
|
+
* NOT a substitute for the log line: an OTel counter is not a backstop for
|
|
40
|
+
* collapsed volume. `registerOTel()` on web is called with no metric readers and
|
|
41
|
+
* the worker only builds one when AXIOM_METRICS_DATASET is set (it is not), so
|
|
42
|
+
* every recordTelemetryCounter call in this repo is a no-op today. If a signal
|
|
43
|
+
* matters it stays in the LOG — which is what `occurrences`/`suppressed` are for.
|
|
44
|
+
*/
|
|
45
|
+
export function createLogCollapser(options) {
|
|
46
|
+
const { windowMs, delaysMs, maxEntries, maxLabels } = options;
|
|
47
|
+
const states = new Map();
|
|
48
|
+
// Same arithmetic as the `startOfWindow` helpers in provider-rate-limits and the
|
|
49
|
+
// managed-spend limiter, so a collapser sharing a window with one of those
|
|
50
|
+
// buckets rolls at exactly the same instant.
|
|
51
|
+
function windowStartFor(nowMs) {
|
|
52
|
+
return Math.floor(nowMs / windowMs) * windowMs;
|
|
53
|
+
}
|
|
54
|
+
// An exhausted schedule repeats its last delay forever. With no schedule at all
|
|
55
|
+
// the key emits once and then waits out the window rather than emitting freely —
|
|
56
|
+
// the safe direction for a primitive whose entire job is to stop a flood.
|
|
57
|
+
function nextDelayMs(emitted) {
|
|
58
|
+
return delaysMs[emitted - 1] ?? delaysMs[delaysMs.length - 1] ?? windowMs;
|
|
59
|
+
}
|
|
60
|
+
function trackLabel(state, label) {
|
|
61
|
+
if (label === undefined || state.labels.has(label))
|
|
62
|
+
return;
|
|
63
|
+
if (state.labels.size >= maxLabels) {
|
|
64
|
+
state.labelsTruncated = true;
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
state.labels.add(label);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Closing totals for a window being retired. Returns null when the last emitted
|
|
71
|
+
* line already carried the window's full occurrence count (`suppressed` back at
|
|
72
|
+
* 0), because then the rollup would only duplicate it.
|
|
73
|
+
*/
|
|
74
|
+
function rollupFor(key, state, reason) {
|
|
75
|
+
if (state.suppressed === 0)
|
|
76
|
+
return null;
|
|
77
|
+
return {
|
|
78
|
+
key,
|
|
79
|
+
windowStartMs: state.windowStartMs,
|
|
80
|
+
occurrences: state.occurrences,
|
|
81
|
+
suppressed: state.suppressed,
|
|
82
|
+
emitted: state.emitted,
|
|
83
|
+
labels: [...state.labels].sort(),
|
|
84
|
+
labelsTruncated: state.labelsTruncated,
|
|
85
|
+
reason,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Make room for a key the map does NOT hold yet. Drops states whose window has
|
|
90
|
+
* already rolled first, then the least-recently-active one, and hands back the
|
|
91
|
+
* closing totals of everything it dropped so eviction can never silently swallow
|
|
92
|
+
* suppressed volume.
|
|
93
|
+
*
|
|
94
|
+
* Eviction is by last activity, not by insertion order: insertion order picks the
|
|
95
|
+
* key that started flooding EARLIEST — the heaviest offender — whose very next
|
|
96
|
+
* occurrence would re-insert and emit again, degrading the collapse back toward
|
|
97
|
+
* per-call emission.
|
|
98
|
+
*/
|
|
99
|
+
function evict(nowMs) {
|
|
100
|
+
const rollups = [];
|
|
101
|
+
if (states.size < maxEntries)
|
|
102
|
+
return rollups;
|
|
103
|
+
for (const [key, state] of states) {
|
|
104
|
+
if (nowMs < state.windowStartMs + windowMs)
|
|
105
|
+
continue;
|
|
106
|
+
const rollup = rollupFor(key, state, "state_evicted");
|
|
107
|
+
if (rollup)
|
|
108
|
+
rollups.push(rollup);
|
|
109
|
+
states.delete(key);
|
|
110
|
+
}
|
|
111
|
+
while (states.size >= maxEntries) {
|
|
112
|
+
let lruKey;
|
|
113
|
+
let lruState;
|
|
114
|
+
for (const [key, state] of states) {
|
|
115
|
+
if (lruState && state.lastSeenMs >= lruState.lastSeenMs)
|
|
116
|
+
continue;
|
|
117
|
+
lruKey = key;
|
|
118
|
+
lruState = state;
|
|
119
|
+
}
|
|
120
|
+
if (lruKey === undefined || lruState === undefined)
|
|
121
|
+
break;
|
|
122
|
+
const rollup = rollupFor(lruKey, lruState, "state_evicted");
|
|
123
|
+
if (rollup)
|
|
124
|
+
rollups.push(rollup);
|
|
125
|
+
states.delete(lruKey);
|
|
126
|
+
}
|
|
127
|
+
return rollups;
|
|
128
|
+
}
|
|
129
|
+
function freshState(label, windowStartMs, nowMs) {
|
|
130
|
+
return {
|
|
131
|
+
windowStartMs,
|
|
132
|
+
occurrences: 1,
|
|
133
|
+
suppressed: 0,
|
|
134
|
+
emitted: 1,
|
|
135
|
+
nextEmitAtMs: nowMs + nextDelayMs(1),
|
|
136
|
+
lastSeenMs: nowMs,
|
|
137
|
+
labels: label === undefined ? new Set() : new Set([label]),
|
|
138
|
+
labelsTruncated: false,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function leadingEdgeEmit(label, windowStartMs) {
|
|
142
|
+
return {
|
|
143
|
+
occurrences: 1,
|
|
144
|
+
suppressed: 0,
|
|
145
|
+
labels: label === undefined ? [] : [label],
|
|
146
|
+
labelsTruncated: false,
|
|
147
|
+
windowStartMs,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
record(key, label) {
|
|
152
|
+
const nowMs = Date.now();
|
|
153
|
+
const windowStartMs = windowStartFor(nowMs);
|
|
154
|
+
const existing = states.get(key);
|
|
155
|
+
// A rolled window starts a fresh line budget. The retired window's collapsed
|
|
156
|
+
// tail (up to a full final-delay's worth of occurrences) is reported as a
|
|
157
|
+
// rollup instead of being overwritten by the fresh state.
|
|
158
|
+
if (existing && existing.windowStartMs !== windowStartMs) {
|
|
159
|
+
const rollup = rollupFor(key, existing, "window_rolled");
|
|
160
|
+
states.set(key, freshState(label, windowStartMs, nowMs));
|
|
161
|
+
return {
|
|
162
|
+
emit: leadingEdgeEmit(label, windowStartMs),
|
|
163
|
+
rollups: rollup ? [rollup] : [],
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
// A key the map does not hold yet is the only case that grows the map, so a
|
|
167
|
+
// key that already exists never costs an unrelated live key its state.
|
|
168
|
+
if (!existing) {
|
|
169
|
+
const rollups = evict(nowMs);
|
|
170
|
+
states.set(key, freshState(label, windowStartMs, nowMs));
|
|
171
|
+
return { emit: leadingEdgeEmit(label, windowStartMs), rollups };
|
|
172
|
+
}
|
|
173
|
+
existing.occurrences += 1;
|
|
174
|
+
existing.lastSeenMs = nowMs;
|
|
175
|
+
trackLabel(existing, label);
|
|
176
|
+
if (nowMs < existing.nextEmitAtMs) {
|
|
177
|
+
existing.suppressed += 1;
|
|
178
|
+
return { emit: null, rollups: [] };
|
|
179
|
+
}
|
|
180
|
+
const suppressed = existing.suppressed;
|
|
181
|
+
existing.suppressed = 0;
|
|
182
|
+
existing.emitted += 1;
|
|
183
|
+
existing.nextEmitAtMs = nowMs + nextDelayMs(existing.emitted);
|
|
184
|
+
return {
|
|
185
|
+
emit: {
|
|
186
|
+
occurrences: existing.occurrences,
|
|
187
|
+
suppressed,
|
|
188
|
+
labels: [...existing.labels].sort(),
|
|
189
|
+
labelsTruncated: existing.labelsTruncated,
|
|
190
|
+
windowStartMs,
|
|
191
|
+
},
|
|
192
|
+
rollups: [],
|
|
193
|
+
};
|
|
194
|
+
},
|
|
195
|
+
reset() {
|
|
196
|
+
states.clear();
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
// ---------------------------------------------------------------------------
|
|
201
|
+
// Once-per-key-per-window claims
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
/** One day. The window a state-transition line almost always wants. */
|
|
204
|
+
const DEFAULT_CLAIM_WINDOW_MS = 24 * 60 * 60_000;
|
|
205
|
+
/**
|
|
206
|
+
* Bounded because the slot key embeds a window that rolls: without the cap a
|
|
207
|
+
* long-lived worker process accumulates one entry per key per window forever.
|
|
208
|
+
*/
|
|
209
|
+
const CLAIM_SLOTS_MAX = 5_000;
|
|
210
|
+
const claimedSlots = new Set();
|
|
211
|
+
/**
|
|
212
|
+
* Claim the single log slot for `key` in the current window: true the first time,
|
|
213
|
+
* false for every repeat until the window rolls.
|
|
214
|
+
*
|
|
215
|
+
* The cheaper sibling of `createLogCollapser` and the right tool when the line
|
|
216
|
+
* reports a STATE TRANSITION rather than a rate — a budget threshold crossed, a
|
|
217
|
+
* connector going unhealthy, a floor being hit. Those lines answer "is this
|
|
218
|
+
* workspace in that state today", so a count adds nothing and a per-call emission
|
|
219
|
+
* is pure flood (the shape that has blinded this dataset before). When the count
|
|
220
|
+
* IS the point, use the collapser, which keeps `occurrences`/`suppressed`.
|
|
221
|
+
*
|
|
222
|
+
* Modelled on `claimThresholdLogSlot`
|
|
223
|
+
* (packages/integrations/src/budget-policy.ts), including its accepted trade: at
|
|
224
|
+
* the bound the set is cleared wholesale rather than evicted one by one, so a key
|
|
225
|
+
* that was already claimed can emit a second line in the same window. Bounded
|
|
226
|
+
* memory beats exactly-once for a line that is idempotent to re-read.
|
|
227
|
+
*/
|
|
228
|
+
export function claimOncePerWindow(key, windowMs = DEFAULT_CLAIM_WINDOW_MS) {
|
|
229
|
+
// NUL separator: it cannot appear in a slug, a UUID, or a decimal window index,
|
|
230
|
+
// so no two distinct (key, window) pairs can collide into one slot.
|
|
231
|
+
const slot = `${key}\u0000${Math.floor(Date.now() / windowMs)}`;
|
|
232
|
+
if (claimedSlots.has(slot))
|
|
233
|
+
return false;
|
|
234
|
+
if (claimedSlots.size >= CLAIM_SLOTS_MAX)
|
|
235
|
+
claimedSlots.clear();
|
|
236
|
+
claimedSlots.add(slot);
|
|
237
|
+
return true;
|
|
238
|
+
}
|
|
239
|
+
/** Test-only: clear the per-process once-per-window claims between cases. */
|
|
240
|
+
export function resetOncePerWindowClaims() {
|
|
241
|
+
claimedSlots.clear();
|
|
242
|
+
}
|
|
@@ -16,6 +16,7 @@ export type LogContext = {
|
|
|
16
16
|
};
|
|
17
17
|
export declare function withLogContext<T>(ctx: LogContext, fn: () => T): T;
|
|
18
18
|
export declare function enterLogContext(ctx: LogContext): void;
|
|
19
|
+
export declare function logLevelForHttpStatus(status: number): LogLevel;
|
|
19
20
|
export type LogSink = (record: Record<string, unknown>) => void;
|
|
20
21
|
export declare function setLogSink(sink: LogSink | null): void;
|
|
21
22
|
export declare function log(level: LogLevel, msg: string, fields?: Record<string, unknown>): void;
|
|
@@ -57,6 +57,54 @@ function reserveSurfaceDimension(merged, ctx) {
|
|
|
57
57
|
surface: ctx.surface,
|
|
58
58
|
};
|
|
59
59
|
}
|
|
60
|
+
// `org_id` is the dimension every org-bucketed monitor, dashboard, and cost query
|
|
61
|
+
// groups on — but `organization_id` is an allowlisted, live column too, and a call
|
|
62
|
+
// site stamps whichever of the two names it learned from its neighbours. Nothing
|
|
63
|
+
// resolves the pair, so the choice is per-call-site and total: measured over 2 days
|
|
64
|
+
// the org_id presence of a given `msg` is either 0% or 100%, never in between,
|
|
65
|
+
// because it is hand-stamped rather than ambient.
|
|
66
|
+
//
|
|
67
|
+
// The consequence is that the biggest "org-less" lines are not org-less at all —
|
|
68
|
+
// they stamp the OTHER name, and every dashboard filtering `org_id` silently misses
|
|
69
|
+
// them: provider.byok_daily_cap_would_block 37,702 rows at 100% organization_id /
|
|
70
|
+
// 0% org_id, credit_commitment_reconcile.billing_owner_unresolved 35,185,
|
|
71
|
+
// ai_column.web_grounding.provider_error 2,351, email_inbox_mirror.row_failed 1,536,
|
|
72
|
+
// managed_ai.timing 1,466, whatsapp.quota_denied 1,392, linkedin.quota_denied 835 —
|
|
73
|
+
// ~80k rows per 2 days.
|
|
74
|
+
//
|
|
75
|
+
// So `organization_id` MIRRORS into `org_id`. Fill only: an org_id already stamped
|
|
76
|
+
// by the call site or inherited from the ambient context wins (it is the narrower,
|
|
77
|
+
// deliberate value), and `organization_id` is never deleted — existing queries and
|
|
78
|
+
// monitors reading it keep working. Both names are already in AXIOM_STABLE_FIELDS
|
|
79
|
+
// with live columns, so this adds ZERO new field names; an allowlisted name without
|
|
80
|
+
// a column rejects the entire ingest batch (the Aug 2026 six-day blackout).
|
|
81
|
+
function reserveOrgDimension(merged) {
|
|
82
|
+
const orgId = merged.org_id;
|
|
83
|
+
// Only genuinely absent/empty counts as fillable. Any other existing value —
|
|
84
|
+
// including a non-string one — is left exactly as the caller wrote it, so this
|
|
85
|
+
// can never rewrite a dimension somebody set on purpose.
|
|
86
|
+
if (!(orgId === undefined || orgId === null || orgId === ""))
|
|
87
|
+
return merged;
|
|
88
|
+
const organizationId = merged.organization_id;
|
|
89
|
+
if (typeof organizationId !== "string" || organizationId.length === 0)
|
|
90
|
+
return merged;
|
|
91
|
+
return { ...merged, org_id: organizationId };
|
|
92
|
+
}
|
|
93
|
+
// Canonical severity for an HTTP status that is being logged. 4xx outcomes are
|
|
94
|
+
// expected client conditions and a 503 is a known-degraded/retryable state (a tenant
|
|
95
|
+
// mid-migration, a provider backing off), not a server fault — so they log as warn
|
|
96
|
+
// noise, and only genuine unexpected failures reach the error stream that monitors
|
|
97
|
+
// page on (OXY-78).
|
|
98
|
+
//
|
|
99
|
+
// Lifted verbatim from `tableRouteLogLevelForStatus`
|
|
100
|
+
// (apps/web/src/lib/tables/route-error-response.ts), which shaped this rule for the
|
|
101
|
+
// table read routes. It lives here so every surface that maps a status to a level
|
|
102
|
+
// agrees on the 503 carve-out instead of re-deriving it — divergence is how
|
|
103
|
+
// `tables.views.list.failed` logged a mid-migration tenant at error while its /rows
|
|
104
|
+
// and /activity siblings logged the same tenant at warn.
|
|
105
|
+
export function logLevelForHttpStatus(status) {
|
|
106
|
+
return status >= 500 && status !== 503 ? "error" : "warn";
|
|
107
|
+
}
|
|
60
108
|
// The sink registration must live on globalThis, not in a module-local variable:
|
|
61
109
|
// Next.js compiles instrumentation.ts and each route handler into separate
|
|
62
110
|
// bundles, each with its OWN instance of this module. A module-local set by the
|
|
@@ -118,7 +166,7 @@ export function log(level, msg, fields) {
|
|
|
118
166
|
sha: process.env.VERCEL_GIT_COMMIT_SHA ?? process.env.OXYGEN_GIT_SHA ?? null,
|
|
119
167
|
region: process.env.VERCEL_REGION ?? null,
|
|
120
168
|
env: process.env.VERCEL_ENV ?? process.env.NODE_ENV ?? null,
|
|
121
|
-
...sanitizeLogFields(reserveSurfaceDimension({ ...context, ...fields }, context)),
|
|
169
|
+
...sanitizeLogFields(reserveOrgDimension(reserveSurfaceDimension({ ...context, ...fields }, context))),
|
|
122
170
|
};
|
|
123
171
|
const line = JSON.stringify(record);
|
|
124
172
|
if (level === "error") {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const OXYGEN_VERSION = "1.
|
|
1
|
+
export declare const OXYGEN_VERSION = "1.804.2";
|
|
2
2
|
export declare const OXYGEN_MINIMUM_CLI_VERSION = "1.181.0";
|
|
3
3
|
export declare const MANAGED_INBOX_MINIMUM_CLI_VERSION = "1.326.2";
|
|
4
4
|
export declare const SUPPORT_AGENT_REPLY_MINIMUM_CLI_VERSION = "1.747.0";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export const OXYGEN_VERSION = "1.
|
|
1
|
+
export const OXYGEN_VERSION = "1.804.2";
|
|
2
2
|
// The GLOBAL CLI compatibility floor: the oldest CLI allowed to call any
|
|
3
3
|
// operational route. Raising it hard-rejects every older CLI from the entire
|
|
4
4
|
// product, so it obeys one law, enforced by scripts/ci/cli-min-version-gate.mjs:
|