instar 0.28.107 → 0.28.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/monitoring/BurnDetector.d.ts +84 -0
- package/dist/monitoring/BurnDetector.d.ts.map +1 -0
- package/dist/monitoring/BurnDetector.js +169 -0
- package/dist/monitoring/BurnDetector.js.map +1 -0
- package/dist/monitoring/BurnThrottleRunbook.d.ts +97 -0
- package/dist/monitoring/BurnThrottleRunbook.d.ts.map +1 -0
- package/dist/monitoring/BurnThrottleRunbook.js +202 -0
- package/dist/monitoring/BurnThrottleRunbook.js.map +1 -0
- package/dist/monitoring/LlmRateGate.d.ts +133 -33
- package/dist/monitoring/LlmRateGate.d.ts.map +1 -1
- package/dist/monitoring/LlmRateGate.js +179 -46
- package/dist/monitoring/LlmRateGate.js.map +1 -1
- package/dist/monitoring/TokenLedger.d.ts +16 -0
- package/dist/monitoring/TokenLedger.d.ts.map +1 -1
- package/dist/monitoring/TokenLedger.js +28 -0
- package/dist/monitoring/TokenLedger.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/0.28.107.md +17 -12
- package/upgrades/0.28.108.md +50 -0
- package/upgrades/side-effects/token-burn-detection-phase-3.md +59 -0
- package/upgrades/side-effects/token-burn-detection-phase-4.md +78 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BurnDetector — emits a structured signal when a single attribution_key
|
|
3
|
+
* crosses configured spend thresholds.
|
|
4
|
+
*
|
|
5
|
+
* Phase 3 of docs/specs/token-burn-detection-and-self-heal.md.
|
|
6
|
+
*
|
|
7
|
+
* Signal-only (per umbrella spec §"Signal-vs-Authority Decomposition"): the
|
|
8
|
+
* detector emits to the existing DegradationReporter. The Phase 4 runbook
|
|
9
|
+
* (Tier-2 Remediator surface) is the only authority that decides whether
|
|
10
|
+
* to alert / throttle / both. The BurnDetector cannot decide anything.
|
|
11
|
+
*
|
|
12
|
+
* Triggers (OR'd):
|
|
13
|
+
* 1. Absolute share — a single attribution_key consumed > absoluteShareThreshold
|
|
14
|
+
* of total 24h spend.
|
|
15
|
+
* 2. Baseline divergence — the key's last-1h rate is > rollingBaselineMultiplier ×
|
|
16
|
+
* its trailing-7-day median rate, AND the 1h rate exceeds rollingBaselineFloor
|
|
17
|
+
* tokens/h.
|
|
18
|
+
*
|
|
19
|
+
* Cold-start (per umbrella spec §"Threshold logic"): the baseline-divergence
|
|
20
|
+
* trigger only fires after a key has been observed for 7 days. Until then,
|
|
21
|
+
* absolute-share is the only signal — that's the trigger that would have
|
|
22
|
+
* caught the 2026-05-15 incident.
|
|
23
|
+
*
|
|
24
|
+
* Polling cadence: 60s, decoupled from the ledger writer. The detector does
|
|
25
|
+
* its own clock (injected `now`) so tests don't need to fake timers globally.
|
|
26
|
+
*/
|
|
27
|
+
import type { TokenLedger } from './TokenLedger.js';
|
|
28
|
+
import type { DegradationReporter } from './DegradationReporter.js';
|
|
29
|
+
export interface BurnDetectionConfig {
|
|
30
|
+
enabled: boolean;
|
|
31
|
+
/** Default 0.25 (25%). */
|
|
32
|
+
absoluteShareThreshold: number;
|
|
33
|
+
/** Default 2 (2x). */
|
|
34
|
+
rollingBaselineMultiplier: number;
|
|
35
|
+
/** Default 10_000_000 tokens/hour. */
|
|
36
|
+
rollingBaselineFloor: number;
|
|
37
|
+
/** Per-key alert cooldown (default 3_600_000ms = 1h). */
|
|
38
|
+
perKeyAlertCooldownMs: number;
|
|
39
|
+
/** Poll interval (default 60_000ms = 60s). */
|
|
40
|
+
pollIntervalMs: number;
|
|
41
|
+
/** Cold-start duration (default 7d). Baseline-divergence trigger disabled until elapsed. */
|
|
42
|
+
coldStartMs: number;
|
|
43
|
+
}
|
|
44
|
+
export declare const DEFAULT_BURN_DETECTION_CONFIG: BurnDetectionConfig;
|
|
45
|
+
export interface BurnSignal {
|
|
46
|
+
attributionKey: string;
|
|
47
|
+
trigger: 'absolute-share' | 'baseline-divergence';
|
|
48
|
+
emittedAt: string;
|
|
49
|
+
observed: {
|
|
50
|
+
tokens24h: number;
|
|
51
|
+
share24h: number;
|
|
52
|
+
tokensLast1h: number;
|
|
53
|
+
projectedDaily: number;
|
|
54
|
+
};
|
|
55
|
+
/** Trailing-7d median tokens/h. Only populated for `baseline-divergence`. */
|
|
56
|
+
baselineMedian7d?: number;
|
|
57
|
+
}
|
|
58
|
+
export interface BurnDetectorDeps {
|
|
59
|
+
ledger: Pick<TokenLedger, 'byAttributionKey' | 'summary'>;
|
|
60
|
+
reporter: Pick<DegradationReporter, 'report'>;
|
|
61
|
+
config?: Partial<BurnDetectionConfig>;
|
|
62
|
+
/** Injectable clock for tests. Defaults to `Date.now`. */
|
|
63
|
+
now?: () => number;
|
|
64
|
+
}
|
|
65
|
+
export declare class BurnDetector {
|
|
66
|
+
private readonly ledger;
|
|
67
|
+
private readonly reporter;
|
|
68
|
+
private readonly config;
|
|
69
|
+
private readonly now;
|
|
70
|
+
/** First time a given attribution_key was seen. Used for cold-start cutoff. */
|
|
71
|
+
private readonly firstSeen;
|
|
72
|
+
/** Last alert emit time per key — gates per-key alert cooldown. */
|
|
73
|
+
private readonly lastAlertAt;
|
|
74
|
+
private timer;
|
|
75
|
+
constructor(deps: BurnDetectorDeps);
|
|
76
|
+
start(): void;
|
|
77
|
+
stop(): void;
|
|
78
|
+
/**
|
|
79
|
+
* One detector cycle. Returns the signals emitted on this tick — useful
|
|
80
|
+
* for tests, and the production loop ignores the return.
|
|
81
|
+
*/
|
|
82
|
+
tick(): BurnSignal[];
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=BurnDetector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BurnDetector.d.ts","sourceRoot":"","sources":["../../src/monitoring/BurnDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAqB,MAAM,kBAAkB,CAAC;AACvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpE,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,0BAA0B;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB;IACtB,yBAAyB,EAAE,MAAM,CAAC;IAClC,sCAAsC;IACtC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,yDAAyD;IACzD,qBAAqB,EAAE,MAAM,CAAC;IAC9B,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4FAA4F;IAC5F,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,6BAA6B,EAAE,mBAQ3C,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,gBAAgB,GAAG,qBAAqB,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,kBAAkB,GAAG,SAAS,CAAC,CAAC;IAC1D,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACtC,0DAA0D;IAC1D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,mEAAmE;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,KAAK,CAA+C;gBAEhD,IAAI,EAAE,gBAAgB;IAOlC,KAAK,IAAI,IAAI;IAgBb,IAAI,IAAI,IAAI;IAOZ;;;OAGG;IACH,IAAI,IAAI,UAAU,EAAE;CAkGrB"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BurnDetector — emits a structured signal when a single attribution_key
|
|
3
|
+
* crosses configured spend thresholds.
|
|
4
|
+
*
|
|
5
|
+
* Phase 3 of docs/specs/token-burn-detection-and-self-heal.md.
|
|
6
|
+
*
|
|
7
|
+
* Signal-only (per umbrella spec §"Signal-vs-Authority Decomposition"): the
|
|
8
|
+
* detector emits to the existing DegradationReporter. The Phase 4 runbook
|
|
9
|
+
* (Tier-2 Remediator surface) is the only authority that decides whether
|
|
10
|
+
* to alert / throttle / both. The BurnDetector cannot decide anything.
|
|
11
|
+
*
|
|
12
|
+
* Triggers (OR'd):
|
|
13
|
+
* 1. Absolute share — a single attribution_key consumed > absoluteShareThreshold
|
|
14
|
+
* of total 24h spend.
|
|
15
|
+
* 2. Baseline divergence — the key's last-1h rate is > rollingBaselineMultiplier ×
|
|
16
|
+
* its trailing-7-day median rate, AND the 1h rate exceeds rollingBaselineFloor
|
|
17
|
+
* tokens/h.
|
|
18
|
+
*
|
|
19
|
+
* Cold-start (per umbrella spec §"Threshold logic"): the baseline-divergence
|
|
20
|
+
* trigger only fires after a key has been observed for 7 days. Until then,
|
|
21
|
+
* absolute-share is the only signal — that's the trigger that would have
|
|
22
|
+
* caught the 2026-05-15 incident.
|
|
23
|
+
*
|
|
24
|
+
* Polling cadence: 60s, decoupled from the ledger writer. The detector does
|
|
25
|
+
* its own clock (injected `now`) so tests don't need to fake timers globally.
|
|
26
|
+
*/
|
|
27
|
+
export const DEFAULT_BURN_DETECTION_CONFIG = {
|
|
28
|
+
enabled: true,
|
|
29
|
+
absoluteShareThreshold: 0.25,
|
|
30
|
+
rollingBaselineMultiplier: 2,
|
|
31
|
+
rollingBaselineFloor: 10_000_000,
|
|
32
|
+
perKeyAlertCooldownMs: 3_600_000,
|
|
33
|
+
pollIntervalMs: 60_000,
|
|
34
|
+
coldStartMs: 7 * 24 * 60 * 60 * 1000,
|
|
35
|
+
};
|
|
36
|
+
export class BurnDetector {
|
|
37
|
+
ledger;
|
|
38
|
+
reporter;
|
|
39
|
+
config;
|
|
40
|
+
now;
|
|
41
|
+
/** First time a given attribution_key was seen. Used for cold-start cutoff. */
|
|
42
|
+
firstSeen = new Map();
|
|
43
|
+
/** Last alert emit time per key — gates per-key alert cooldown. */
|
|
44
|
+
lastAlertAt = new Map();
|
|
45
|
+
timer = null;
|
|
46
|
+
constructor(deps) {
|
|
47
|
+
this.ledger = deps.ledger;
|
|
48
|
+
this.reporter = deps.reporter;
|
|
49
|
+
this.config = { ...DEFAULT_BURN_DETECTION_CONFIG, ...(deps.config ?? {}) };
|
|
50
|
+
this.now = deps.now ?? (() => Date.now());
|
|
51
|
+
}
|
|
52
|
+
start() {
|
|
53
|
+
if (!this.config.enabled || this.timer)
|
|
54
|
+
return;
|
|
55
|
+
this.timer = setInterval(() => {
|
|
56
|
+
try {
|
|
57
|
+
this.tick();
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
console.warn(`[burn-detector] tick error (non-fatal): ${err.message}`);
|
|
61
|
+
}
|
|
62
|
+
}, this.config.pollIntervalMs);
|
|
63
|
+
// Prevent the interval from keeping the process alive on its own — the
|
|
64
|
+
// server's lifecycle owns shutdown.
|
|
65
|
+
if (this.timer && typeof this.timer.unref === 'function') {
|
|
66
|
+
this.timer.unref();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
stop() {
|
|
70
|
+
if (this.timer) {
|
|
71
|
+
clearInterval(this.timer);
|
|
72
|
+
this.timer = null;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* One detector cycle. Returns the signals emitted on this tick — useful
|
|
77
|
+
* for tests, and the production loop ignores the return.
|
|
78
|
+
*/
|
|
79
|
+
tick() {
|
|
80
|
+
if (!this.config.enabled)
|
|
81
|
+
return [];
|
|
82
|
+
const now = this.now();
|
|
83
|
+
const since24h = now - 24 * 60 * 60 * 1000;
|
|
84
|
+
const since1h = now - 60 * 60 * 1000;
|
|
85
|
+
const since7d = now - 7 * 24 * 60 * 60 * 1000;
|
|
86
|
+
const keys24h = this.ledger.byAttributionKey({ sinceMs: since24h });
|
|
87
|
+
if (keys24h.length === 0)
|
|
88
|
+
return [];
|
|
89
|
+
const total24h = keys24h.reduce((sum, k) => sum + k.totalTokens, 0);
|
|
90
|
+
if (total24h <= 0)
|
|
91
|
+
return [];
|
|
92
|
+
// Update first-seen map.
|
|
93
|
+
for (const k of keys24h) {
|
|
94
|
+
if (!this.firstSeen.has(k.attributionKey)) {
|
|
95
|
+
this.firstSeen.set(k.attributionKey, k.firstTs);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const signals = [];
|
|
99
|
+
const keys1h = this.ledger.byAttributionKey({ sinceMs: since1h });
|
|
100
|
+
const keys1hMap = new Map(keys1h.map((k) => [k.attributionKey, k]));
|
|
101
|
+
for (const key24h of keys24h) {
|
|
102
|
+
const share = key24h.totalTokens / total24h;
|
|
103
|
+
const key1h = keys1hMap.get(key24h.attributionKey);
|
|
104
|
+
const tokens1h = key1h ? key1h.totalTokens : 0;
|
|
105
|
+
const projectedDaily = tokens1h * 24;
|
|
106
|
+
// Skip exempt runbook self-attribution prefix (defence in depth — Phase 4
|
|
107
|
+
// runbook also exempts itself at the gate level).
|
|
108
|
+
if (key24h.attributionKey.startsWith('burn-throttle-runbook::'))
|
|
109
|
+
continue;
|
|
110
|
+
// Cooldown.
|
|
111
|
+
const lastAlert = this.lastAlertAt.get(key24h.attributionKey) ?? 0;
|
|
112
|
+
if (now - lastAlert < this.config.perKeyAlertCooldownMs)
|
|
113
|
+
continue;
|
|
114
|
+
let trigger = null;
|
|
115
|
+
let baselineMedian7d;
|
|
116
|
+
// Trigger 1 — absolute share.
|
|
117
|
+
if (share > this.config.absoluteShareThreshold) {
|
|
118
|
+
trigger = 'absolute-share';
|
|
119
|
+
}
|
|
120
|
+
// Trigger 2 — baseline divergence (only after cold-start window).
|
|
121
|
+
if (!trigger) {
|
|
122
|
+
const firstSeen = this.firstSeen.get(key24h.attributionKey) ?? key24h.firstTs;
|
|
123
|
+
if (now - firstSeen >= this.config.coldStartMs && tokens1h >= this.config.rollingBaselineFloor) {
|
|
124
|
+
// 7-day median rate (tokens/h). Use the simple per-key 7d total / 168
|
|
125
|
+
// as a proxy when the table doesn't store hourly buckets — Phase 4 may
|
|
126
|
+
// refine to a true median.
|
|
127
|
+
const keys7d = this.ledger.byAttributionKey({ sinceMs: since7d });
|
|
128
|
+
const key7d = keys7d.find((k) => k.attributionKey === key24h.attributionKey);
|
|
129
|
+
if (key7d) {
|
|
130
|
+
baselineMedian7d = key7d.totalTokens / (7 * 24);
|
|
131
|
+
if (tokens1h > this.config.rollingBaselineMultiplier * baselineMedian7d) {
|
|
132
|
+
trigger = 'baseline-divergence';
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (!trigger)
|
|
138
|
+
continue;
|
|
139
|
+
const signal = {
|
|
140
|
+
attributionKey: key24h.attributionKey,
|
|
141
|
+
trigger,
|
|
142
|
+
emittedAt: new Date(now).toISOString(),
|
|
143
|
+
observed: {
|
|
144
|
+
tokens24h: key24h.totalTokens,
|
|
145
|
+
share24h: share,
|
|
146
|
+
tokensLast1h: tokens1h,
|
|
147
|
+
projectedDaily,
|
|
148
|
+
},
|
|
149
|
+
baselineMedian7d,
|
|
150
|
+
};
|
|
151
|
+
signals.push(signal);
|
|
152
|
+
this.lastAlertAt.set(key24h.attributionKey, now);
|
|
153
|
+
// Emit to DegradationReporter. The runbook in Phase 4 will subscribe via
|
|
154
|
+
// the existing Remediator dispatch and decide alert vs throttle.
|
|
155
|
+
this.reporter.report({
|
|
156
|
+
feature: 'token-burn-detection',
|
|
157
|
+
primary: `attribution_key ${key24h.attributionKey} sustained spend within thresholds`,
|
|
158
|
+
fallback: `signal-only: detector flagged the key (Phase 4 runbook will decide alert vs throttle)`,
|
|
159
|
+
reason: trigger === 'absolute-share'
|
|
160
|
+
? `${key24h.attributionKey} consumed ${(share * 100).toFixed(1)}% of 24h spend (threshold ${(this.config.absoluteShareThreshold * 100).toFixed(0)}%)`
|
|
161
|
+
: `${key24h.attributionKey} last-1h rate ${tokens1h.toLocaleString()} tok/h, baseline ${baselineMedian7d?.toLocaleString() ?? '?'} tok/h (multiplier ${this.config.rollingBaselineMultiplier}x)`,
|
|
162
|
+
impact: `Projected ${projectedDaily.toLocaleString()} tokens in next 24h at current rate. ` +
|
|
163
|
+
`Phase 3 is observation-only; Phase 4 wires alerting and bounded auto-throttle.`,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
return signals;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=BurnDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BurnDetector.js","sourceRoot":"","sources":["../../src/monitoring/BurnDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAqBH,MAAM,CAAC,MAAM,6BAA6B,GAAwB;IAChE,OAAO,EAAE,IAAI;IACb,sBAAsB,EAAE,IAAI;IAC5B,yBAAyB,EAAE,CAAC;IAC5B,oBAAoB,EAAE,UAAU;IAChC,qBAAqB,EAAE,SAAS;IAChC,cAAc,EAAE,MAAM;IACtB,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;CACrC,CAAC;AAwBF,MAAM,OAAO,YAAY;IACN,MAAM,CAA6B;IACnC,QAAQ,CAA+B;IACvC,MAAM,CAAsB;IAC5B,GAAG,CAAe;IACnC,+EAA+E;IAC9D,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvD,mEAAmE;IAClD,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,KAAK,GAA0C,IAAI,CAAC;IAE5D,YAAY,IAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,6BAA6B,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAC3E,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QAC/C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,2CAA4C,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACpF,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/B,uEAAuE;QACvE,oCAAoC;QACpC,IAAI,IAAI,CAAC,KAAK,IAAI,OAAQ,IAAI,CAAC,KAAgC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACpF,IAAI,CAAC,KAA+B,CAAC,KAAK,EAAE,CAAC;QAChD,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAE9C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACpE,IAAI,QAAQ,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAE7B,yBAAyB;QACzB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;YAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,cAAc,GAAG,QAAQ,GAAG,EAAE,CAAC;YAErC,0EAA0E;YAC1E,kDAAkD;YAClD,IAAI,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,yBAAyB,CAAC;gBAAE,SAAS;YAE1E,YAAY;YACZ,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACnE,IAAI,GAAG,GAAG,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB;gBAAE,SAAS;YAElE,IAAI,OAAO,GAAiC,IAAI,CAAC;YACjD,IAAI,gBAAoC,CAAC;YAEzC,8BAA8B;YAC9B,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;gBAC/C,OAAO,GAAG,gBAAgB,CAAC;YAC7B,CAAC;YAED,kEAAkE;YAClE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC;gBAC9E,IAAI,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;oBAC/F,sEAAsE;oBACtE,uEAAuE;oBACvE,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;oBAClE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,MAAM,CAAC,cAAc,CAAC,CAAC;oBAC7E,IAAI,KAAK,EAAE,CAAC;wBACV,gBAAgB,GAAG,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;wBAChD,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,GAAG,gBAAgB,EAAE,CAAC;4BACxE,OAAO,GAAG,qBAAqB,CAAC;wBAClC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,MAAM,MAAM,GAAe;gBACzB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,OAAO;gBACP,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE;gBACtC,QAAQ,EAAE;oBACR,SAAS,EAAE,MAAM,CAAC,WAAW;oBAC7B,QAAQ,EAAE,KAAK;oBACf,YAAY,EAAE,QAAQ;oBACtB,cAAc;iBACf;gBACD,gBAAgB;aACjB,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAEjD,yEAAyE;YACzE,iEAAiE;YACjE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACnB,OAAO,EAAE,sBAAsB;gBAC/B,OAAO,EAAE,mBAAmB,MAAM,CAAC,cAAc,oCAAoC;gBACrF,QAAQ,EAAE,uFAAuF;gBACjG,MAAM,EACJ,OAAO,KAAK,gBAAgB;oBAC1B,CAAC,CAAC,GAAG,MAAM,CAAC,cAAc,aAAa,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;oBACrJ,CAAC,CAAC,GAAG,MAAM,CAAC,cAAc,iBAAiB,QAAQ,CAAC,cAAc,EAAE,oBAAoB,gBAAgB,EAAE,cAAc,EAAE,IAAI,GAAG,sBAAsB,IAAI,CAAC,MAAM,CAAC,yBAAyB,IAAI;gBACpM,MAAM,EACJ,aAAa,cAAc,CAAC,cAAc,EAAE,uCAAuC;oBACnF,gFAAgF;aACnF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BurnThrottleRunbook — Tier-2 authority surface for burn-detection signals.
|
|
3
|
+
*
|
|
4
|
+
* Phase 4 of docs/specs/token-burn-detection-and-self-heal.md. This is the
|
|
5
|
+
* ONLY blocking surface in the burn-detection pipeline. The BurnDetector
|
|
6
|
+
* (Phase 3) emits structured signals via DegradationReporter; this runbook
|
|
7
|
+
* subscribes via the existing handler hook and decides:
|
|
8
|
+
* - alert-only (default for `unknown::*` keys and any caller config disables)
|
|
9
|
+
* - throttle (default for known component names)
|
|
10
|
+
* - both (the umbrella spec's primary policy)
|
|
11
|
+
*
|
|
12
|
+
* Authority shape (per umbrella spec §"Signal-vs-Authority Decomposition"):
|
|
13
|
+
* the runbook is the only piece in this pipeline with blocking authority.
|
|
14
|
+
* The detector emits; the gate enforces; the runbook decides.
|
|
15
|
+
*
|
|
16
|
+
* Self-reinforcing-loop guard: the runbook never throttles a key starting
|
|
17
|
+
* with `burn-throttle-runbook::*` — defence-in-depth (the gate also refuses
|
|
18
|
+
* to install such throttles, the detector also exempts the prefix). This
|
|
19
|
+
* three-layer guard is from the convergence audit.
|
|
20
|
+
*/
|
|
21
|
+
import { LlmRateGate, type InstalledThrottle } from './LlmRateGate.js';
|
|
22
|
+
import type { DegradationEvent } from './DegradationReporter.js';
|
|
23
|
+
export interface BurnThrottleConfig {
|
|
24
|
+
/** Whether to auto-throttle (true) or alert-only (false). Default true. */
|
|
25
|
+
autoThrottle: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Whether to auto-throttle keys with no known component (`unknown::*`).
|
|
28
|
+
* Default false — the umbrella spec is conservative here: an unknown key
|
|
29
|
+
* may be a user extension the agent doesn't recognise, and throttling
|
|
30
|
+
* something the agent doesn't understand can break user workflows.
|
|
31
|
+
* Operators opt in to auto-throttle on unknown explicitly.
|
|
32
|
+
*/
|
|
33
|
+
autoThrottleOnUnknown: boolean;
|
|
34
|
+
/** Throttle duration (default 60min). */
|
|
35
|
+
throttleDurationMs: number;
|
|
36
|
+
}
|
|
37
|
+
export declare const DEFAULT_BURN_THROTTLE_CONFIG: BurnThrottleConfig;
|
|
38
|
+
export type RunbookOutcomeKind = 'alert-only-unknown' | 'alert-only-config-disabled' | 'alert-only-self-attribution' | 'throttle-installed' | 'throttle-failed';
|
|
39
|
+
export interface RunbookOutcome {
|
|
40
|
+
kind: RunbookOutcomeKind;
|
|
41
|
+
attributionKey: string;
|
|
42
|
+
decidedAt: string;
|
|
43
|
+
trigger: string;
|
|
44
|
+
/** When kind === 'throttle-installed', the actually-installed throttle. */
|
|
45
|
+
throttle?: InstalledThrottle;
|
|
46
|
+
/** Free-text reason for the decision. */
|
|
47
|
+
reason: string;
|
|
48
|
+
}
|
|
49
|
+
/** Telegram emit function. Phase 5 upgrades to principal-bound buttons. */
|
|
50
|
+
export type TelegramAlertSender = (topicId: number, text: string) => void | Promise<void>;
|
|
51
|
+
export interface BurnThrottleRunbookDeps {
|
|
52
|
+
gate: LlmRateGate;
|
|
53
|
+
config?: Partial<BurnThrottleConfig>;
|
|
54
|
+
/** Telegram emit. Optional — when missing, decisions still happen, just no alert. */
|
|
55
|
+
sendTelegram?: TelegramAlertSender;
|
|
56
|
+
/** Telegram topic ID for alerts. */
|
|
57
|
+
alertTopicId?: number;
|
|
58
|
+
/** Injectable clock for tests. */
|
|
59
|
+
now?: () => number;
|
|
60
|
+
}
|
|
61
|
+
/** Identity tag the runbook uses when installing throttles. */
|
|
62
|
+
export declare const RUNBOOK_ISSUER = "burn-throttle-runbook";
|
|
63
|
+
export declare class BurnThrottleRunbook {
|
|
64
|
+
private readonly gate;
|
|
65
|
+
private readonly config;
|
|
66
|
+
private readonly sendTelegram;
|
|
67
|
+
private readonly alertTopicId;
|
|
68
|
+
private readonly now;
|
|
69
|
+
constructor(deps: BurnThrottleRunbookDeps);
|
|
70
|
+
/**
|
|
71
|
+
* Handle a burn-detection signal. Called from a subscriber hook on
|
|
72
|
+
* DegradationReporter for feature='token-burn-detection' events.
|
|
73
|
+
*
|
|
74
|
+
* Returns the outcome so tests + audit + the verification step (Phase 6)
|
|
75
|
+
* can inspect what happened.
|
|
76
|
+
*/
|
|
77
|
+
handle(event: DegradationEvent): RunbookOutcome;
|
|
78
|
+
private fireTelegram;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Extract the attribution key from a burn-detection DegradationEvent. The
|
|
82
|
+
* BurnDetector formats the `primary` field as "attribution_key <KEY>
|
|
83
|
+
* sustained spend within thresholds" — we parse it back here. The wire
|
|
84
|
+
* shape is internal to the burn-detection pipeline; if it changes, change
|
|
85
|
+
* both ends in the same PR.
|
|
86
|
+
*/
|
|
87
|
+
export declare function extractAttributionKey(event: DegradationEvent): string;
|
|
88
|
+
/**
|
|
89
|
+
* Extract the trigger label from the event's reason field. Returns
|
|
90
|
+
* 'absolute-share' or 'baseline-divergence' or 'unknown-trigger'.
|
|
91
|
+
*/
|
|
92
|
+
export declare function extractTrigger(event: DegradationEvent): string;
|
|
93
|
+
/** Is the attribution key one of the unknown::* fallback shapes? */
|
|
94
|
+
export declare function isUnknownKey(attributionKey: string): boolean;
|
|
95
|
+
/** Compose the ELI16 alert text for Telegram. No backticks, no camelCase config keys, narrative tone. */
|
|
96
|
+
export declare function composeAlertText(event: DegradationEvent, attributionKey: string, trigger: string): string;
|
|
97
|
+
//# sourceMappingURL=BurnThrottleRunbook.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BurnThrottleRunbook.d.ts","sourceRoot":"","sources":["../../src/monitoring/BurnThrottleRunbook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,MAAM,WAAW,kBAAkB;IACjC,2EAA2E;IAC3E,YAAY,EAAE,OAAO,CAAC;IACtB;;;;;;OAMG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,yCAAyC;IACzC,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,eAAO,MAAM,4BAA4B,EAAE,kBAI1C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAC1B,oBAAoB,GACpB,4BAA4B,GAC5B,6BAA6B,GAC7B,oBAAoB,GACpB,iBAAiB,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,kBAAkB,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,2EAA2E;AAC3E,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1F,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,WAAW,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACrC,qFAAqF;IACrF,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,+DAA+D;AAC/D,eAAO,MAAM,cAAc,0BAA0B,CAAC;AAEtD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAc;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAC/D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;gBAEvB,IAAI,EAAE,uBAAuB;IAQzC;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,EAAE,gBAAgB,GAAG,cAAc;IA8F/C,OAAO,CAAC,YAAY;CAYrB;AAID;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,gBAAgB,GAAG,MAAM,CAGrE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,gBAAgB,GAAG,MAAM,CAI9D;AAED,oEAAoE;AACpE,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED,yGAAyG;AACzG,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAazG"}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BurnThrottleRunbook — Tier-2 authority surface for burn-detection signals.
|
|
3
|
+
*
|
|
4
|
+
* Phase 4 of docs/specs/token-burn-detection-and-self-heal.md. This is the
|
|
5
|
+
* ONLY blocking surface in the burn-detection pipeline. The BurnDetector
|
|
6
|
+
* (Phase 3) emits structured signals via DegradationReporter; this runbook
|
|
7
|
+
* subscribes via the existing handler hook and decides:
|
|
8
|
+
* - alert-only (default for `unknown::*` keys and any caller config disables)
|
|
9
|
+
* - throttle (default for known component names)
|
|
10
|
+
* - both (the umbrella spec's primary policy)
|
|
11
|
+
*
|
|
12
|
+
* Authority shape (per umbrella spec §"Signal-vs-Authority Decomposition"):
|
|
13
|
+
* the runbook is the only piece in this pipeline with blocking authority.
|
|
14
|
+
* The detector emits; the gate enforces; the runbook decides.
|
|
15
|
+
*
|
|
16
|
+
* Self-reinforcing-loop guard: the runbook never throttles a key starting
|
|
17
|
+
* with `burn-throttle-runbook::*` — defence-in-depth (the gate also refuses
|
|
18
|
+
* to install such throttles, the detector also exempts the prefix). This
|
|
19
|
+
* three-layer guard is from the convergence audit.
|
|
20
|
+
*/
|
|
21
|
+
export const DEFAULT_BURN_THROTTLE_CONFIG = {
|
|
22
|
+
autoThrottle: true,
|
|
23
|
+
autoThrottleOnUnknown: false,
|
|
24
|
+
throttleDurationMs: 60 * 60 * 1000,
|
|
25
|
+
};
|
|
26
|
+
/** Identity tag the runbook uses when installing throttles. */
|
|
27
|
+
export const RUNBOOK_ISSUER = 'burn-throttle-runbook';
|
|
28
|
+
export class BurnThrottleRunbook {
|
|
29
|
+
gate;
|
|
30
|
+
config;
|
|
31
|
+
sendTelegram;
|
|
32
|
+
alertTopicId;
|
|
33
|
+
now;
|
|
34
|
+
constructor(deps) {
|
|
35
|
+
this.gate = deps.gate;
|
|
36
|
+
this.config = { ...DEFAULT_BURN_THROTTLE_CONFIG, ...(deps.config ?? {}) };
|
|
37
|
+
this.sendTelegram = deps.sendTelegram;
|
|
38
|
+
this.alertTopicId = deps.alertTopicId ?? 8615;
|
|
39
|
+
this.now = deps.now ?? (() => Date.now());
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Handle a burn-detection signal. Called from a subscriber hook on
|
|
43
|
+
* DegradationReporter for feature='token-burn-detection' events.
|
|
44
|
+
*
|
|
45
|
+
* Returns the outcome so tests + audit + the verification step (Phase 6)
|
|
46
|
+
* can inspect what happened.
|
|
47
|
+
*/
|
|
48
|
+
handle(event) {
|
|
49
|
+
const attributionKey = extractAttributionKey(event);
|
|
50
|
+
const trigger = extractTrigger(event);
|
|
51
|
+
const decidedAt = new Date(this.now()).toISOString();
|
|
52
|
+
const signalId = `${event.timestamp}:${attributionKey}`;
|
|
53
|
+
// 1. Self-attribution: refuse to throttle the runbook itself, AND emit
|
|
54
|
+
// a high-severity escalation alert. Per Phase 4 second-pass review §3:
|
|
55
|
+
// silent swallow is the wrong response here — if the runbook itself
|
|
56
|
+
// is being attributed-to as a burn source, something is structurally
|
|
57
|
+
// wrong (a bug in the runbook's own LLM-bound code, or attribution
|
|
58
|
+
// being mis-applied to a legitimate runbook call). Either way, that's
|
|
59
|
+
// exactly the case a user needs to know about. So we DO send Telegram.
|
|
60
|
+
if (attributionKey.startsWith(`${RUNBOOK_ISSUER}::`)) {
|
|
61
|
+
this.fireTelegram(`URGENT: the burn-throttle runbook itself is being flagged as a burn source ` +
|
|
62
|
+
`(${attributionKey}). I refused to throttle myself by design, but this likely ` +
|
|
63
|
+
`means either the runbook has a bug producing too many LLM calls, or attribution ` +
|
|
64
|
+
`is being mis-applied to legitimate runbook work. Please investigate.`);
|
|
65
|
+
return {
|
|
66
|
+
kind: 'alert-only-self-attribution',
|
|
67
|
+
attributionKey,
|
|
68
|
+
decidedAt,
|
|
69
|
+
trigger,
|
|
70
|
+
reason: 'Refusing to throttle the runbook itself (self-reinforcing-loop guard); escalation alert sent.',
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
// Compose the alert text now so we send it regardless of throttle outcome.
|
|
74
|
+
const alertText = composeAlertText(event, attributionKey, trigger);
|
|
75
|
+
// 2. Caller config disables auto-throttle.
|
|
76
|
+
if (!this.config.autoThrottle) {
|
|
77
|
+
this.fireTelegram(alertText);
|
|
78
|
+
return {
|
|
79
|
+
kind: 'alert-only-config-disabled',
|
|
80
|
+
attributionKey,
|
|
81
|
+
decidedAt,
|
|
82
|
+
trigger,
|
|
83
|
+
reason: 'autoThrottle is disabled in config; alert sent but no throttle installed.',
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
// 3. Unknown attribution + caller has not opted in to auto-throttle on unknown.
|
|
87
|
+
if (isUnknownKey(attributionKey) && !this.config.autoThrottleOnUnknown) {
|
|
88
|
+
this.fireTelegram(alertText);
|
|
89
|
+
return {
|
|
90
|
+
kind: 'alert-only-unknown',
|
|
91
|
+
attributionKey,
|
|
92
|
+
decidedAt,
|
|
93
|
+
trigger,
|
|
94
|
+
reason: 'Attribution key has no known component; operator must opt in to auto-throttle on unknown.',
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
// 4. Install throttle, send alert.
|
|
98
|
+
const token = this.gate.computeCapabilityToken({
|
|
99
|
+
attributionKey,
|
|
100
|
+
durationMs: this.config.throttleDurationMs,
|
|
101
|
+
issuer: RUNBOOK_ISSUER,
|
|
102
|
+
signalId,
|
|
103
|
+
}) ?? undefined;
|
|
104
|
+
try {
|
|
105
|
+
const throttle = this.gate.installThrottle({
|
|
106
|
+
attributionKey,
|
|
107
|
+
durationMs: this.config.throttleDurationMs,
|
|
108
|
+
reason: `${trigger} trigger crossed threshold; runbook installed bounded throttle.`,
|
|
109
|
+
issuer: RUNBOOK_ISSUER,
|
|
110
|
+
signalId,
|
|
111
|
+
capabilityToken: token,
|
|
112
|
+
});
|
|
113
|
+
this.fireTelegram(`${alertText}\n\nI have slowed this component down. It will resume automatically in ${Math.round(this.config.throttleDurationMs / 60000)} minutes, or you can reply STOP to release it sooner.`);
|
|
114
|
+
return {
|
|
115
|
+
kind: 'throttle-installed',
|
|
116
|
+
attributionKey,
|
|
117
|
+
decidedAt,
|
|
118
|
+
trigger,
|
|
119
|
+
throttle,
|
|
120
|
+
reason: 'Throttle installed, alert sent.',
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
this.fireTelegram(`${alertText}\n\nI tried to slow it down automatically but the slowdown did not take effect. You may need to look at this one.`);
|
|
125
|
+
return {
|
|
126
|
+
kind: 'throttle-failed',
|
|
127
|
+
attributionKey,
|
|
128
|
+
decidedAt,
|
|
129
|
+
trigger,
|
|
130
|
+
reason: `Throttle install failed: ${err.message}`,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
fireTelegram(text) {
|
|
135
|
+
if (!this.sendTelegram)
|
|
136
|
+
return;
|
|
137
|
+
try {
|
|
138
|
+
// Caller may return a promise; we don't await — the runbook is on the
|
|
139
|
+
// signal-handling hot path and must not block. Errors are swallowed:
|
|
140
|
+
// a failed alert can't block the throttle decision, which is the
|
|
141
|
+
// important part.
|
|
142
|
+
void this.sendTelegram(this.alertTopicId, text);
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// Intentional swallow.
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/* ---------- pure helpers (exported for tests) ---------- */
|
|
150
|
+
/**
|
|
151
|
+
* Extract the attribution key from a burn-detection DegradationEvent. The
|
|
152
|
+
* BurnDetector formats the `primary` field as "attribution_key <KEY>
|
|
153
|
+
* sustained spend within thresholds" — we parse it back here. The wire
|
|
154
|
+
* shape is internal to the burn-detection pipeline; if it changes, change
|
|
155
|
+
* both ends in the same PR.
|
|
156
|
+
*/
|
|
157
|
+
export function extractAttributionKey(event) {
|
|
158
|
+
const m = /attribution_key\s+(\S+)/.exec(event.primary || '');
|
|
159
|
+
return m && m[1] ? m[1] : 'unknown::no-key';
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Extract the trigger label from the event's reason field. Returns
|
|
163
|
+
* 'absolute-share' or 'baseline-divergence' or 'unknown-trigger'.
|
|
164
|
+
*/
|
|
165
|
+
export function extractTrigger(event) {
|
|
166
|
+
if (/% of 24h spend/.test(event.reason || ''))
|
|
167
|
+
return 'absolute-share';
|
|
168
|
+
if (/baseline/i.test(event.reason || ''))
|
|
169
|
+
return 'baseline-divergence';
|
|
170
|
+
return 'unknown-trigger';
|
|
171
|
+
}
|
|
172
|
+
/** Is the attribution key one of the unknown::* fallback shapes? */
|
|
173
|
+
export function isUnknownKey(attributionKey) {
|
|
174
|
+
return attributionKey.startsWith('unknown::');
|
|
175
|
+
}
|
|
176
|
+
/** Compose the ELI16 alert text for Telegram. No backticks, no camelCase config keys, narrative tone. */
|
|
177
|
+
export function composeAlertText(event, attributionKey, trigger) {
|
|
178
|
+
const friendlyName = humanize(attributionKey);
|
|
179
|
+
const triggerNarrative = trigger === 'absolute-share'
|
|
180
|
+
? 'is using more than a quarter of the agent\'s total token budget over the last day'
|
|
181
|
+
: trigger === 'baseline-divergence'
|
|
182
|
+
? 'has more than doubled its normal rate in the last hour, and the absolute amount is large enough to matter'
|
|
183
|
+
: 'crossed one of the burn thresholds';
|
|
184
|
+
return (`Heads up: ${friendlyName} ${triggerNarrative}.\n\n` +
|
|
185
|
+
`${event.reason}\n\n` +
|
|
186
|
+
`${event.impact}`);
|
|
187
|
+
}
|
|
188
|
+
function humanize(attributionKey) {
|
|
189
|
+
if (attributionKey.startsWith('unknown::'))
|
|
190
|
+
return 'an unknown component';
|
|
191
|
+
if (attributionKey.startsWith('user-job:')) {
|
|
192
|
+
const m = /^user-job:([^:]+)/.exec(attributionKey);
|
|
193
|
+
return m ? `your scheduled job "${m[1]}"` : 'a scheduled job';
|
|
194
|
+
}
|
|
195
|
+
if (attributionKey.startsWith('user-hook:')) {
|
|
196
|
+
const m = /^user-hook:([^:]+)/.exec(attributionKey);
|
|
197
|
+
return m ? `your hook "${m[1]}"` : 'a hook';
|
|
198
|
+
}
|
|
199
|
+
const m = /^([^:]+)::/.exec(attributionKey);
|
|
200
|
+
return m ? `the ${m[1]} component` : attributionKey;
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=BurnThrottleRunbook.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BurnThrottleRunbook.js","sourceRoot":"","sources":["../../src/monitoring/BurnThrottleRunbook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAoBH,MAAM,CAAC,MAAM,4BAA4B,GAAuB;IAC9D,YAAY,EAAE,IAAI;IAClB,qBAAqB,EAAE,KAAK;IAC5B,kBAAkB,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;CACnC,CAAC;AAkCF,+DAA+D;AAC/D,MAAM,CAAC,MAAM,cAAc,GAAG,uBAAuB,CAAC;AAEtD,MAAM,OAAO,mBAAmB;IACb,IAAI,CAAc;IAClB,MAAM,CAAqB;IAC3B,YAAY,CAAkC;IAC9C,YAAY,CAAS;IACrB,GAAG,CAAe;IAEnC,YAAY,IAA6B;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,4BAA4B,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAuB;QAC5B,MAAM,cAAc,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,SAAS,IAAI,cAAc,EAAE,CAAC;QAExD,uEAAuE;QACvE,0EAA0E;QAC1E,uEAAuE;QACvE,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,cAAc,IAAI,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,YAAY,CACf,6EAA6E;gBAC7E,IAAI,cAAc,6DAA6D;gBAC/E,kFAAkF;gBAClF,sEAAsE,CACvE,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,6BAA6B;gBACnC,cAAc;gBACd,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,+FAA+F;aACxG,CAAC;QACJ,CAAC;QAED,2EAA2E;QAC3E,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEnE,2CAA2C;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,4BAA4B;gBAClC,cAAc;gBACd,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,2EAA2E;aACpF,CAAC;QACJ,CAAC;QAED,gFAAgF;QAChF,IAAI,YAAY,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACvE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,oBAAoB;gBAC1B,cAAc;gBACd,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,2FAA2F;aACpG,CAAC;QACJ,CAAC;QAED,mCAAmC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC;YAC7C,cAAc;YACd,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;YAC1C,MAAM,EAAE,cAAc;YACtB,QAAQ;SACT,CAAC,IAAI,SAAS,CAAC;QAEhB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;gBACzC,cAAc;gBACd,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;gBAC1C,MAAM,EAAE,GAAG,OAAO,iEAAiE;gBACnF,MAAM,EAAE,cAAc;gBACtB,QAAQ;gBACR,eAAe,EAAE,KAAK;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,GAAG,SAAS,0EAA0E,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACnN,OAAO;gBACL,IAAI,EAAE,oBAAoB;gBAC1B,cAAc;gBACd,SAAS;gBACT,OAAO;gBACP,QAAQ;gBACR,MAAM,EAAE,iCAAiC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,CAAC,GAAG,SAAS,mHAAmH,CAAC,CAAC;YACnJ,OAAO;gBACL,IAAI,EAAE,iBAAiB;gBACvB,cAAc;gBACd,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,4BAA6B,GAAa,CAAC,OAAO,EAAE;aAC7D,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAC/B,IAAI,CAAC;YACH,sEAAsE;YACtE,qEAAqE;YACrE,iEAAiE;YACjE,kBAAkB;YAClB,KAAK,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;CACF;AAED,6DAA6D;AAE7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAuB;IAC3D,MAAM,CAAC,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAuB;IACpD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;QAAE,OAAO,gBAAgB,CAAC;IACvE,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;QAAE,OAAO,qBAAqB,CAAC;IACvE,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,YAAY,CAAC,cAAsB;IACjD,OAAO,cAAc,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,gBAAgB,CAAC,KAAuB,EAAE,cAAsB,EAAE,OAAe;IAC/F,MAAM,YAAY,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GACpB,OAAO,KAAK,gBAAgB;QAC1B,CAAC,CAAC,mFAAmF;QACrF,CAAC,CAAC,OAAO,KAAK,qBAAqB;YACnC,CAAC,CAAC,2GAA2G;YAC7G,CAAC,CAAC,oCAAoC,CAAC;IAC3C,OAAO,CACL,aAAa,YAAY,IAAI,gBAAgB,OAAO;QACpD,GAAG,KAAK,CAAC,MAAM,MAAM;QACrB,GAAG,KAAK,CAAC,MAAM,EAAE,CAClB,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,cAAsB;IACtC,IAAI,cAAc,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,sBAAsB,CAAC;IAC1E,IAAI,cAAc,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACnD,OAAO,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAChE,CAAC;IACD,IAAI,cAAc,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,oBAAoB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACpD,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9C,CAAC;IACD,MAAM,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC5C,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC;AACtD,CAAC"}
|