instar 1.3.230 → 1.3.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +29 -1
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +24 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/LlmCircuitBreaker.d.ts +15 -0
- package/dist/core/LlmCircuitBreaker.d.ts.map +1 -1
- package/dist/core/LlmCircuitBreaker.js +37 -1
- package/dist/core/LlmCircuitBreaker.js.map +1 -1
- package/dist/core/types.d.ts +16 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/McpProcessReaper.d.ts +135 -0
- package/dist/monitoring/McpProcessReaper.d.ts.map +1 -0
- package/dist/monitoring/McpProcessReaper.js +229 -0
- package/dist/monitoring/McpProcessReaper.js.map +1 -0
- package/dist/monitoring/ResourceLedger.d.ts +78 -0
- package/dist/monitoring/ResourceLedger.d.ts.map +1 -0
- package/dist/monitoring/ResourceLedger.js +163 -0
- package/dist/monitoring/ResourceLedger.js.map +1 -0
- package/dist/monitoring/ResourceLedgerPoller.d.ts +64 -0
- package/dist/monitoring/ResourceLedgerPoller.d.ts.map +1 -0
- package/dist/monitoring/ResourceLedgerPoller.js +83 -0
- package/dist/monitoring/ResourceLedgerPoller.js.map +1 -0
- package/dist/monitoring/mcpProcessReaperDeps.d.ts +22 -0
- package/dist/monitoring/mcpProcessReaperDeps.d.ts.map +1 -0
- package/dist/monitoring/mcpProcessReaperDeps.js +125 -0
- package/dist/monitoring/mcpProcessReaperDeps.js.map +1 -0
- package/dist/monitoring/mcpProcessSignatures.d.ts +38 -0
- package/dist/monitoring/mcpProcessSignatures.d.ts.map +1 -0
- package/dist/monitoring/mcpProcessSignatures.js +55 -0
- package/dist/monitoring/mcpProcessSignatures.js.map +1 -0
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +5 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/AgentServer.d.ts +5 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +50 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/CapabilityIndex.d.ts.map +1 -1
- package/dist/server/CapabilityIndex.js +12 -0
- package/dist/server/CapabilityIndex.js.map +1 -1
- package/dist/server/routes.d.ts +4 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +31 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +47 -47
- package/src/scaffold/templates.ts +5 -0
- package/upgrades/1.3.231.md +39 -0
- package/upgrades/1.3.232.md +47 -0
- package/upgrades/side-effects/mcp-process-reaper.md +58 -0
- package/upgrades/side-effects/resource-ledger-phase-a.md +101 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* McpProcessReaper — Option B of the MCP-leak fix (Codey's design in
|
|
3
|
+
* `.instar/apprenticeship/codey-task-mcp-leak-reaper.md`).
|
|
4
|
+
*
|
|
5
|
+
* The problem: a session spawns MCP-server children (Playwright, mcp-remote,
|
|
6
|
+
* instar stdio). Killing the session's main pid does NOT cascade to those
|
|
7
|
+
* children — they re-parent (to PID 1 / launchd) or are held by an npm-exec
|
|
8
|
+
* wrapper and survive for days. The fleet accumulated ~80 such leaked procs,
|
|
9
|
+
* up to 5 days old, as a persistent load floor. `OrphanProcessReaper` reaps the
|
|
10
|
+
* session-level CLI proc but not these MCP *descendants*.
|
|
11
|
+
*
|
|
12
|
+
* This is a descendant-aware SWEEP (a sibling reaper, NOT a change to the
|
|
13
|
+
* shared OrphanProcessReaper / ReapGuard path — zero blast radius on the
|
|
14
|
+
* working reaper). For each allow-listed MCP proc it resolves the owning tmux
|
|
15
|
+
* session by walking the ppid chain, and reaps ONLY when the proc is old AND
|
|
16
|
+
* its owning session is a dead/stale *instar* session or fully orphaned. It
|
|
17
|
+
* NEVER touches a proc under a live/tracked session, nor under an external
|
|
18
|
+
* (non-instar) tmux session.
|
|
19
|
+
*
|
|
20
|
+
* Ships DARK + DRY-RUN by default (kills processes — same posture as
|
|
21
|
+
* SessionReaper / AgentWorktreeReaper). Every decision is audited.
|
|
22
|
+
*/
|
|
23
|
+
import { EventEmitter } from 'node:events';
|
|
24
|
+
import { matchMcpSignature, type McpProcessSignature } from './mcpProcessSignatures.js';
|
|
25
|
+
export interface McpProcessReaperConfig {
|
|
26
|
+
enabled: boolean;
|
|
27
|
+
dryRun: boolean;
|
|
28
|
+
/** A proc younger than this is always kept (a starting session's children). */
|
|
29
|
+
minAgeMs: number;
|
|
30
|
+
reapIntervalMs: number;
|
|
31
|
+
/** Bounded blast radius per pass. */
|
|
32
|
+
maxReapsPerPass: number;
|
|
33
|
+
/** Max ppid hops when resolving a proc's owning tmux session. */
|
|
34
|
+
maxAncestorHops: number;
|
|
35
|
+
}
|
|
36
|
+
export declare const DEFAULT_MCP_PROCESS_REAPER_CONFIG: McpProcessReaperConfig;
|
|
37
|
+
export interface McpProcessInfo {
|
|
38
|
+
pid: number;
|
|
39
|
+
ppid: number;
|
|
40
|
+
elapsedMs: number;
|
|
41
|
+
command: string;
|
|
42
|
+
signatureId: McpProcessSignature['id'];
|
|
43
|
+
}
|
|
44
|
+
export type McpVerdict = 'keep' | 'reap-eligible';
|
|
45
|
+
export interface McpEvaluation {
|
|
46
|
+
pid: number;
|
|
47
|
+
signatureId: McpProcessSignature['id'];
|
|
48
|
+
ageMs: number;
|
|
49
|
+
owningSession: string | null;
|
|
50
|
+
verdict: McpVerdict;
|
|
51
|
+
/** The gate that forced KEEP, or the reap rationale. */
|
|
52
|
+
reason: string;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* All signal sources injected so the classifier is unit-testable without a
|
|
56
|
+
* real process table, tmux, or kill(2). Production wiring supplies ps/tmux/
|
|
57
|
+
* SessionManager-backed implementations.
|
|
58
|
+
*/
|
|
59
|
+
export interface McpProcessReaperDeps {
|
|
60
|
+
/** Allow-listed MCP-server processes currently running (this uid). */
|
|
61
|
+
listMcpProcesses: () => McpProcessInfo[];
|
|
62
|
+
/** Full pid → ppid map of the process table (for ancestor resolution). */
|
|
63
|
+
getProcessTree: () => Map<number, number>;
|
|
64
|
+
/** tmux pane pid → session name. */
|
|
65
|
+
getTmuxPaneMap: () => Map<number, string>;
|
|
66
|
+
/** tmux session names that are genuinely live/tracked. SACRED — never reaped. */
|
|
67
|
+
getLiveSessions: () => Set<string>;
|
|
68
|
+
/** All instar-pattern tmux session names present (tracked or not). Used to
|
|
69
|
+
* tell a dead/stale *instar* session (reapable) from an external one (kept). */
|
|
70
|
+
getInstarSessions: () => Set<string>;
|
|
71
|
+
/** SIGTERM the proc. Only called when killsEnabled. */
|
|
72
|
+
killProcess: (pid: number) => void;
|
|
73
|
+
/** Append one audit entry (JSONL). Never throws into the reap loop. */
|
|
74
|
+
audit?: (entry: McpReapAuditEntry) => void;
|
|
75
|
+
now?: () => number;
|
|
76
|
+
}
|
|
77
|
+
export interface McpReapAuditEntry {
|
|
78
|
+
ts: number;
|
|
79
|
+
type: 'reaped' | 'would-reap' | 'kept';
|
|
80
|
+
pid: number;
|
|
81
|
+
signatureId: string;
|
|
82
|
+
ageMs: number;
|
|
83
|
+
owningSession: string | null;
|
|
84
|
+
reason: string;
|
|
85
|
+
dryRun: boolean;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Walk the ppid chain from `startPid` until a pid is a tmux pane pid. Returns
|
|
89
|
+
* the owning tmux session name, or null when no tmux ancestor is found within
|
|
90
|
+
* `maxHops` (i.e. the proc is orphaned / re-parented — its session is dead).
|
|
91
|
+
* Pure — cycle-safe via a visited set.
|
|
92
|
+
*/
|
|
93
|
+
export declare function resolveOwningSession(startPid: number, tree: Map<number, number>, tmuxPaneMap: Map<number, string>, maxHops: number): string | null;
|
|
94
|
+
/**
|
|
95
|
+
* Pure per-proc classifier. KEEP unless the proc is old AND its owning session
|
|
96
|
+
* is a dead/stale instar session or fully orphaned. Order matters: the sacred
|
|
97
|
+
* gates (live session, external session) short-circuit before age is consulted.
|
|
98
|
+
*/
|
|
99
|
+
export declare function classifyMcpProcess(proc: McpProcessInfo, owningSession: string | null, liveSessions: Set<string>, instarSessions: Set<string>, minAgeMs: number): McpEvaluation;
|
|
100
|
+
export declare class McpProcessReaper extends EventEmitter {
|
|
101
|
+
private readonly cfg;
|
|
102
|
+
private readonly deps;
|
|
103
|
+
private readonly now;
|
|
104
|
+
private timer?;
|
|
105
|
+
private running;
|
|
106
|
+
private lastPassAt;
|
|
107
|
+
private reapedLastPass;
|
|
108
|
+
constructor(deps: McpProcessReaperDeps, cfg?: Partial<McpProcessReaperConfig>);
|
|
109
|
+
start(): void;
|
|
110
|
+
stop(): void;
|
|
111
|
+
private get killsEnabled();
|
|
112
|
+
/** Resolve + classify one proc against the current process/tmux/session view. */
|
|
113
|
+
private evaluate;
|
|
114
|
+
/** One reap pass. Returns the per-proc evaluations + what was reaped. */
|
|
115
|
+
reap(): Promise<{
|
|
116
|
+
ts: number;
|
|
117
|
+
evaluations: McpEvaluation[];
|
|
118
|
+
reaped: number[];
|
|
119
|
+
dryRun: boolean;
|
|
120
|
+
}>;
|
|
121
|
+
private writeAudit;
|
|
122
|
+
/** Observability snapshot for GET /processes/mcp-reaper (no side effects). */
|
|
123
|
+
snapshot(): {
|
|
124
|
+
enabled: boolean;
|
|
125
|
+
dryRun: boolean;
|
|
126
|
+
minAgeMs: number;
|
|
127
|
+
lastPassAt: number;
|
|
128
|
+
reapedLastPass: number;
|
|
129
|
+
processes: McpEvaluation[];
|
|
130
|
+
reapEligible: number;
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/** Re-export for production wiring that builds McpProcessInfo from a ps line. */
|
|
134
|
+
export { matchMcpSignature };
|
|
135
|
+
//# sourceMappingURL=McpProcessReaper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"McpProcessReaper.d.ts","sourceRoot":"","sources":["../../src/monitoring/McpProcessReaper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAExF,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,qCAAqC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,iCAAiC,EAAE,sBAS/C,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,mBAAmB,CAAC,IAAI,CAAC,CAAC;CACxC;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,eAAe,CAAC;AAElD,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,UAAU,CAAC;IACpB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,sEAAsE;IACtE,gBAAgB,EAAE,MAAM,cAAc,EAAE,CAAC;IACzC,0EAA0E;IAC1E,cAAc,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,oCAAoC;IACpC,cAAc,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,iFAAiF;IACjF,eAAe,EAAE,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC;qFACiF;IACjF,iBAAiB,EAAE,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,uDAAuD;IACvD,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,uEAAuE;IACvE,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC3C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,QAAQ,GAAG,YAAY,GAAG,MAAM,CAAC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACzB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,IAAI,CAWf;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,cAAc,EACpB,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,EACzB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,QAAQ,EAAE,MAAM,GACf,aAAa,CAyBf;AAED,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,KAAK,CAAC,CAAiB;IAC/B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,cAAc,CAAK;gBAEf,IAAI,EAAE,oBAAoB,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC;IAO7E,KAAK,IAAI,IAAI;IAMb,IAAI,IAAI,IAAI;IAIZ,OAAO,KAAK,YAAY,GAEvB;IAED,iFAAiF;IACjF,OAAO,CAAC,QAAQ;IAWhB,yEAAyE;IACnE,IAAI,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAiEtG,OAAO,CAAC,UAAU;IAIlB,8EAA8E;IAC9E,QAAQ,IAAI;QACV,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QACpD,UAAU,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAC3C,SAAS,EAAE,aAAa,EAAE,CAAC;QAC3B,YAAY,EAAE,MAAM,CAAC;KACtB;CAyBF;AAED,iFAAiF;AACjF,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* McpProcessReaper — Option B of the MCP-leak fix (Codey's design in
|
|
3
|
+
* `.instar/apprenticeship/codey-task-mcp-leak-reaper.md`).
|
|
4
|
+
*
|
|
5
|
+
* The problem: a session spawns MCP-server children (Playwright, mcp-remote,
|
|
6
|
+
* instar stdio). Killing the session's main pid does NOT cascade to those
|
|
7
|
+
* children — they re-parent (to PID 1 / launchd) or are held by an npm-exec
|
|
8
|
+
* wrapper and survive for days. The fleet accumulated ~80 such leaked procs,
|
|
9
|
+
* up to 5 days old, as a persistent load floor. `OrphanProcessReaper` reaps the
|
|
10
|
+
* session-level CLI proc but not these MCP *descendants*.
|
|
11
|
+
*
|
|
12
|
+
* This is a descendant-aware SWEEP (a sibling reaper, NOT a change to the
|
|
13
|
+
* shared OrphanProcessReaper / ReapGuard path — zero blast radius on the
|
|
14
|
+
* working reaper). For each allow-listed MCP proc it resolves the owning tmux
|
|
15
|
+
* session by walking the ppid chain, and reaps ONLY when the proc is old AND
|
|
16
|
+
* its owning session is a dead/stale *instar* session or fully orphaned. It
|
|
17
|
+
* NEVER touches a proc under a live/tracked session, nor under an external
|
|
18
|
+
* (non-instar) tmux session.
|
|
19
|
+
*
|
|
20
|
+
* Ships DARK + DRY-RUN by default (kills processes — same posture as
|
|
21
|
+
* SessionReaper / AgentWorktreeReaper). Every decision is audited.
|
|
22
|
+
*/
|
|
23
|
+
import { EventEmitter } from 'node:events';
|
|
24
|
+
import { matchMcpSignature } from './mcpProcessSignatures.js';
|
|
25
|
+
export const DEFAULT_MCP_PROCESS_REAPER_CONFIG = {
|
|
26
|
+
enabled: false,
|
|
27
|
+
dryRun: true,
|
|
28
|
+
// Conservative: the leaked procs are hours-to-days old; 2h floor means a
|
|
29
|
+
// legitimately busy short-lived MCP child is never a candidate.
|
|
30
|
+
minAgeMs: 2 * 3600 * 1000,
|
|
31
|
+
reapIntervalMs: 30 * 60 * 1000,
|
|
32
|
+
maxReapsPerPass: 25,
|
|
33
|
+
maxAncestorHops: 30,
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Walk the ppid chain from `startPid` until a pid is a tmux pane pid. Returns
|
|
37
|
+
* the owning tmux session name, or null when no tmux ancestor is found within
|
|
38
|
+
* `maxHops` (i.e. the proc is orphaned / re-parented — its session is dead).
|
|
39
|
+
* Pure — cycle-safe via a visited set.
|
|
40
|
+
*/
|
|
41
|
+
export function resolveOwningSession(startPid, tree, tmuxPaneMap, maxHops) {
|
|
42
|
+
let pid = startPid;
|
|
43
|
+
const seen = new Set();
|
|
44
|
+
for (let hop = 0; hop <= maxHops && pid !== undefined && pid > 1; hop++) {
|
|
45
|
+
if (seen.has(pid))
|
|
46
|
+
break; // cycle guard
|
|
47
|
+
seen.add(pid);
|
|
48
|
+
const session = tmuxPaneMap.get(pid);
|
|
49
|
+
if (session)
|
|
50
|
+
return session;
|
|
51
|
+
pid = tree.get(pid);
|
|
52
|
+
}
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Pure per-proc classifier. KEEP unless the proc is old AND its owning session
|
|
57
|
+
* is a dead/stale instar session or fully orphaned. Order matters: the sacred
|
|
58
|
+
* gates (live session, external session) short-circuit before age is consulted.
|
|
59
|
+
*/
|
|
60
|
+
export function classifyMcpProcess(proc, owningSession, liveSessions, instarSessions, minAgeMs) {
|
|
61
|
+
const base = {
|
|
62
|
+
pid: proc.pid,
|
|
63
|
+
signatureId: proc.signatureId,
|
|
64
|
+
ageMs: proc.elapsedMs,
|
|
65
|
+
owningSession,
|
|
66
|
+
};
|
|
67
|
+
const keep = (reason) => ({ ...base, verdict: 'keep', reason });
|
|
68
|
+
const reap = (reason) => ({ ...base, verdict: 'reap-eligible', reason });
|
|
69
|
+
if (owningSession !== null) {
|
|
70
|
+
// SACRED: a live/tracked session owns it — keep regardless of age. A long-
|
|
71
|
+
// running autonomous session legitimately owns old MCP servers.
|
|
72
|
+
if (liveSessions.has(owningSession))
|
|
73
|
+
return keep('session-live');
|
|
74
|
+
// An external (non-instar) tmux session — never touch the user's processes.
|
|
75
|
+
if (!instarSessions.has(owningSession))
|
|
76
|
+
return keep('external-session');
|
|
77
|
+
// A stale/dead *instar* session still lingering in tmux: reapable when old.
|
|
78
|
+
if (proc.elapsedMs < minAgeMs)
|
|
79
|
+
return keep('stale-instar-too-young');
|
|
80
|
+
return reap(`stale-instar-session:${owningSession}`);
|
|
81
|
+
}
|
|
82
|
+
// No tmux ancestor — the owning session is dead and this proc re-parented.
|
|
83
|
+
// This is the dominant leak shape. Reapable only when old.
|
|
84
|
+
if (proc.elapsedMs < minAgeMs)
|
|
85
|
+
return keep('orphan-too-young');
|
|
86
|
+
return reap('orphaned-no-session');
|
|
87
|
+
}
|
|
88
|
+
export class McpProcessReaper extends EventEmitter {
|
|
89
|
+
cfg;
|
|
90
|
+
deps;
|
|
91
|
+
now;
|
|
92
|
+
timer;
|
|
93
|
+
running = false;
|
|
94
|
+
lastPassAt = 0;
|
|
95
|
+
reapedLastPass = 0;
|
|
96
|
+
constructor(deps, cfg) {
|
|
97
|
+
super();
|
|
98
|
+
this.deps = deps;
|
|
99
|
+
this.cfg = { ...DEFAULT_MCP_PROCESS_REAPER_CONFIG, ...(cfg ?? {}) };
|
|
100
|
+
this.now = deps.now ?? (() => Date.now());
|
|
101
|
+
}
|
|
102
|
+
start() {
|
|
103
|
+
if (this.timer || !this.cfg.enabled)
|
|
104
|
+
return;
|
|
105
|
+
this.timer = setInterval(() => { void this.reap(); }, this.cfg.reapIntervalMs);
|
|
106
|
+
if (typeof this.timer.unref === 'function')
|
|
107
|
+
this.timer.unref();
|
|
108
|
+
}
|
|
109
|
+
stop() {
|
|
110
|
+
if (this.timer) {
|
|
111
|
+
clearInterval(this.timer);
|
|
112
|
+
this.timer = undefined;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
get killsEnabled() {
|
|
116
|
+
return this.cfg.enabled && !this.cfg.dryRun;
|
|
117
|
+
}
|
|
118
|
+
/** Resolve + classify one proc against the current process/tmux/session view. */
|
|
119
|
+
evaluate(proc, tree, tmuxPaneMap, liveSessions, instarSessions) {
|
|
120
|
+
const owningSession = resolveOwningSession(proc.pid, tree, tmuxPaneMap, this.cfg.maxAncestorHops);
|
|
121
|
+
return classifyMcpProcess(proc, owningSession, liveSessions, instarSessions, this.cfg.minAgeMs);
|
|
122
|
+
}
|
|
123
|
+
/** One reap pass. Returns the per-proc evaluations + what was reaped. */
|
|
124
|
+
async reap() {
|
|
125
|
+
if (this.running)
|
|
126
|
+
return { ts: this.now(), evaluations: [], reaped: [], dryRun: !this.killsEnabled };
|
|
127
|
+
this.running = true;
|
|
128
|
+
const reaped = [];
|
|
129
|
+
const evaluations = [];
|
|
130
|
+
try {
|
|
131
|
+
let procs;
|
|
132
|
+
let tree;
|
|
133
|
+
let tmuxPaneMap;
|
|
134
|
+
let liveSessions;
|
|
135
|
+
let instarSessions;
|
|
136
|
+
try {
|
|
137
|
+
procs = this.deps.listMcpProcesses();
|
|
138
|
+
tree = this.deps.getProcessTree();
|
|
139
|
+
tmuxPaneMap = this.deps.getTmuxPaneMap();
|
|
140
|
+
liveSessions = this.deps.getLiveSessions();
|
|
141
|
+
instarSessions = this.deps.getInstarSessions();
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
this.emit('error', err);
|
|
145
|
+
return { ts: this.now(), evaluations: [], reaped: [], dryRun: !this.killsEnabled };
|
|
146
|
+
}
|
|
147
|
+
for (const proc of procs) {
|
|
148
|
+
let evaln;
|
|
149
|
+
try {
|
|
150
|
+
evaln = this.evaluate(proc, tree, tmuxPaneMap, liveSessions, instarSessions);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
// A signal threw — cannot reason about it, so KEEP. Never reap on a
|
|
154
|
+
// failed evaluation.
|
|
155
|
+
evaln = {
|
|
156
|
+
pid: proc.pid, signatureId: proc.signatureId, ageMs: proc.elapsedMs,
|
|
157
|
+
owningSession: null, verdict: 'keep', reason: 'eval-error',
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
evaluations.push(evaln);
|
|
161
|
+
if (evaln.verdict !== 'reap-eligible') {
|
|
162
|
+
this.writeAudit({ ts: this.now(), type: 'kept', pid: proc.pid, signatureId: proc.signatureId, ageMs: proc.elapsedMs, owningSession: evaln.owningSession, reason: evaln.reason, dryRun: !this.killsEnabled });
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
if (reaped.length >= this.cfg.maxReapsPerPass)
|
|
166
|
+
continue; // blast-radius cap
|
|
167
|
+
if (!this.killsEnabled) {
|
|
168
|
+
// Dry-run: classify + audit what we WOULD kill, kill nothing.
|
|
169
|
+
this.writeAudit({ ts: this.now(), type: 'would-reap', pid: proc.pid, signatureId: proc.signatureId, ageMs: proc.elapsedMs, owningSession: evaln.owningSession, reason: evaln.reason, dryRun: true });
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
this.deps.killProcess(proc.pid);
|
|
174
|
+
reaped.push(proc.pid);
|
|
175
|
+
this.writeAudit({ ts: this.now(), type: 'reaped', pid: proc.pid, signatureId: proc.signatureId, ageMs: proc.elapsedMs, owningSession: evaln.owningSession, reason: evaln.reason, dryRun: false });
|
|
176
|
+
this.emit('reaped', proc);
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
this.emit('error', err);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
this.lastPassAt = this.now();
|
|
183
|
+
this.reapedLastPass = reaped.length;
|
|
184
|
+
this.emit('pass', { evaluations, reaped });
|
|
185
|
+
}
|
|
186
|
+
finally {
|
|
187
|
+
this.running = false;
|
|
188
|
+
}
|
|
189
|
+
return { ts: this.now(), evaluations, reaped, dryRun: !this.killsEnabled };
|
|
190
|
+
}
|
|
191
|
+
writeAudit(entry) {
|
|
192
|
+
try {
|
|
193
|
+
this.deps.audit?.(entry);
|
|
194
|
+
}
|
|
195
|
+
catch { /* audit must never break the loop */ }
|
|
196
|
+
}
|
|
197
|
+
/** Observability snapshot for GET /processes/mcp-reaper (no side effects). */
|
|
198
|
+
snapshot() {
|
|
199
|
+
let processes = [];
|
|
200
|
+
try {
|
|
201
|
+
const procs = this.deps.listMcpProcesses();
|
|
202
|
+
const tree = this.deps.getProcessTree();
|
|
203
|
+
const tmuxPaneMap = this.deps.getTmuxPaneMap();
|
|
204
|
+
const liveSessions = this.deps.getLiveSessions();
|
|
205
|
+
const instarSessions = this.deps.getInstarSessions();
|
|
206
|
+
processes = procs.map((proc) => {
|
|
207
|
+
try {
|
|
208
|
+
return this.evaluate(proc, tree, tmuxPaneMap, liveSessions, instarSessions);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return { pid: proc.pid, signatureId: proc.signatureId, ageMs: proc.elapsedMs, owningSession: null, verdict: 'keep', reason: 'eval-error' };
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
catch { /* listing failed — report empty, never crash the route */ }
|
|
216
|
+
return {
|
|
217
|
+
enabled: this.cfg.enabled,
|
|
218
|
+
dryRun: this.cfg.dryRun,
|
|
219
|
+
minAgeMs: this.cfg.minAgeMs,
|
|
220
|
+
lastPassAt: this.lastPassAt,
|
|
221
|
+
reapedLastPass: this.reapedLastPass,
|
|
222
|
+
processes,
|
|
223
|
+
reapEligible: processes.filter((p) => p.verdict === 'reap-eligible').length,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
/** Re-export for production wiring that builds McpProcessInfo from a ps line. */
|
|
228
|
+
export { matchMcpSignature };
|
|
229
|
+
//# sourceMappingURL=McpProcessReaper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"McpProcessReaper.js","sourceRoot":"","sources":["../../src/monitoring/McpProcessReaper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAA4B,MAAM,2BAA2B,CAAC;AAcxF,MAAM,CAAC,MAAM,iCAAiC,GAA2B;IACvE,OAAO,EAAE,KAAK;IACd,MAAM,EAAE,IAAI;IACZ,yEAAyE;IACzE,gEAAgE;IAChE,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;IACzB,cAAc,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;IAC9B,eAAe,EAAE,EAAE;IACnB,eAAe,EAAE,EAAE;CACpB,CAAC;AAyDF;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAAgB,EAChB,IAAyB,EACzB,WAAgC,EAChC,OAAe;IAEf,IAAI,GAAG,GAAuB,QAAQ,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,OAAO,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,cAAc;QACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAoB,EACpB,aAA4B,EAC5B,YAAyB,EACzB,cAA2B,EAC3B,QAAgB;IAEhB,MAAM,IAAI,GAAG;QACX,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,IAAI,CAAC,SAAS;QACrB,aAAa;KACd,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,MAAc,EAAiB,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,CAAC,MAAc,EAAiB,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC;IAEhG,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QAC3B,2EAA2E;QAC3E,gEAAgE;QAChE,IAAI,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,OAAO,IAAI,CAAC,cAAc,CAAC,CAAC;QACjE,4EAA4E;QAC5E,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACxE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS,GAAG,QAAQ;YAAE,OAAO,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC,wBAAwB,aAAa,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,2EAA2E;IAC3E,2DAA2D;IAC3D,IAAI,IAAI,CAAC,SAAS,GAAG,QAAQ;QAAE,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAAY;IAC/B,GAAG,CAAyB;IAC5B,IAAI,CAAuB;IAC3B,GAAG,CAAe;IAC3B,KAAK,CAAkB;IACvB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,CAAC,CAAC;IACf,cAAc,GAAG,CAAC,CAAC;IAE3B,YAAY,IAA0B,EAAE,GAAqC;QAC3E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,EAAE,GAAG,iCAAiC,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;QACpE,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO;QAC5C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU;YAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACjE,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QAAC,CAAC;IACxE,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,iFAAiF;IACzE,QAAQ,CACd,IAAoB,EACpB,IAAyB,EACzB,WAAgC,EAChC,YAAyB,EACzB,cAA2B;QAE3B,MAAM,aAAa,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAClG,OAAO,kBAAkB,CAAC,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClG,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QACrG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAoB,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,KAAuB,CAAC;YAC5B,IAAI,IAAyB,CAAC;YAC9B,IAAI,WAAgC,CAAC;YACrC,IAAI,YAAyB,CAAC;YAC9B,IAAI,cAA2B,CAAC;YAChC,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACrC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACzC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC3C,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrF,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,KAAoB,CAAC;gBACzB,IAAI,CAAC;oBACH,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;gBAC/E,CAAC;gBAAC,MAAM,CAAC;oBACP,oEAAoE;oBACpE,qBAAqB;oBACrB,KAAK,GAAG;wBACN,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS;wBACnE,aAAa,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY;qBAC3D,CAAC;gBACJ,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAExB,IAAI,KAAK,CAAC,OAAO,KAAK,eAAe,EAAE,CAAC;oBACtC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;oBAC7M,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,eAAe;oBAAE,SAAS,CAAC,mBAAmB;gBAE5E,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACvB,8DAA8D;oBAC9D,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;oBACrM,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC;oBACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACtB,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBAClM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7C,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;IAC7E,CAAC;IAEO,UAAU,CAAC,KAAwB;QACzC,IAAI,CAAC;YAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,qCAAqC,CAAC,CAAC;IACnF,CAAC;IAED,8EAA8E;IAC9E,QAAQ;QAMN,IAAI,SAAS,GAAoB,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrD,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC7B,IAAI,CAAC;oBAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;gBAAC,CAAC;gBACpF,MAAM,CAAC;oBACL,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,EAAE,OAAO,EAAE,MAAoB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;gBAC3J,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC,CAAC,0DAA0D,CAAC,CAAC;QACtE,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO;YACzB,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YACvB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ;YAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,SAAS;YACT,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,eAAe,CAAC,CAAC,MAAM;SAC5E,CAAC;IACJ,CAAC;CACF;AAED,iFAAiF;AACjF,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { Database as BetterSqliteDatabase } from 'better-sqlite3';
|
|
2
|
+
/** Canonical event kinds. `circuit-open`/`circuit-recover` come from the breaker
|
|
3
|
+
* (the primary, account-level signal); the throttle/quota/529 kinds come from
|
|
4
|
+
* RateLimitSentinel (session-scoped, secondary — tagged `source`). */
|
|
5
|
+
export type RateLimitEventKind = 'circuit-open' | 'circuit-recover' | 'throttle' | 'quota' | '529';
|
|
6
|
+
export type RateLimitEventSource = 'circuit-breaker' | 'session-sentinel';
|
|
7
|
+
export interface RateLimitEventInput {
|
|
8
|
+
ts: number;
|
|
9
|
+
kind: RateLimitEventKind;
|
|
10
|
+
source: RateLimitEventSource;
|
|
11
|
+
/** Stable per-event sequence within this process — together with ts+source it
|
|
12
|
+
* forms the row id, so same-ms events are distinct and restarts don't replay. */
|
|
13
|
+
seq: number;
|
|
14
|
+
accountKey?: string;
|
|
15
|
+
sessionName?: string;
|
|
16
|
+
reason?: string;
|
|
17
|
+
detail?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface RateLimitSummaryRow {
|
|
20
|
+
/** Headline: breaker trips (the account-level signal). */
|
|
21
|
+
circuitOpenCount: number;
|
|
22
|
+
circuitRecoverCount: number;
|
|
23
|
+
/** Session-sentinel detections, reported separately (different semantics). */
|
|
24
|
+
sentinelCount: number;
|
|
25
|
+
totalEvents: number;
|
|
26
|
+
/** Breaker trips per hour over the window — the "rate" asked for. */
|
|
27
|
+
tripsPerHour: number;
|
|
28
|
+
oldestEventTs: number | null;
|
|
29
|
+
newestEventTs: number | null;
|
|
30
|
+
windowMs: number;
|
|
31
|
+
}
|
|
32
|
+
export interface RateLimitKindRow {
|
|
33
|
+
kind: RateLimitEventKind;
|
|
34
|
+
source: RateLimitEventSource;
|
|
35
|
+
count: number;
|
|
36
|
+
firstTs: number;
|
|
37
|
+
lastTs: number;
|
|
38
|
+
}
|
|
39
|
+
export interface RateLimitEventRow {
|
|
40
|
+
id: string;
|
|
41
|
+
ts: number;
|
|
42
|
+
kind: RateLimitEventKind;
|
|
43
|
+
source: RateLimitEventSource;
|
|
44
|
+
accountKey: string;
|
|
45
|
+
sessionName: string | null;
|
|
46
|
+
reason: string | null;
|
|
47
|
+
detail: string | null;
|
|
48
|
+
}
|
|
49
|
+
export interface ResourceLedgerOptions {
|
|
50
|
+
/** SQLite db path, or ':memory:' for tests. */
|
|
51
|
+
dbPath: string;
|
|
52
|
+
/** Test seam — inject a Database instance. */
|
|
53
|
+
databaseFactory?: (dbPath: string) => BetterSqliteDatabase;
|
|
54
|
+
}
|
|
55
|
+
export declare class ResourceLedger {
|
|
56
|
+
private db;
|
|
57
|
+
private insertEvent;
|
|
58
|
+
private closed;
|
|
59
|
+
constructor(opts: ResourceLedgerOptions);
|
|
60
|
+
/** Per-event identity: ts + source + seq is unique within a process; the source
|
|
61
|
+
* prefix keeps breaker and sentinel rows from ever colliding. NOT a content
|
|
62
|
+
* hash, so two genuine same-ms events stay distinct. */
|
|
63
|
+
private static eventId;
|
|
64
|
+
/** Record one rate-limit event. Idempotent on (source,ts,seq). Never throws —
|
|
65
|
+
* observability must not break the observed path. */
|
|
66
|
+
recordRateLimitEvent(e: RateLimitEventInput): void;
|
|
67
|
+
/** Count + rate of rate-limit events over the trailing window. */
|
|
68
|
+
rateLimitSummary(nowMs: number, windowMs: number): RateLimitSummaryRow;
|
|
69
|
+
/** Per-kind breakdown over the window. */
|
|
70
|
+
rateLimitByKind(nowMs: number, windowMs: number): RateLimitKindRow[];
|
|
71
|
+
/** Recent event rows, newest first. */
|
|
72
|
+
rateLimitEvents(opts?: {
|
|
73
|
+
sinceMs?: number;
|
|
74
|
+
limit?: number;
|
|
75
|
+
}): RateLimitEventRow[];
|
|
76
|
+
close(): void;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=ResourceLedger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ResourceLedger.d.ts","sourceRoot":"","sources":["../../src/monitoring/ResourceLedger.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,QAAQ,IAAI,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAIvE;;uEAEuE;AACvE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,iBAAiB,GACjB,UAAU,GACV,OAAO,GACP,KAAK,CAAC;AAEV,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;AAE1E,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B;sFACkF;IAClF,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACpC,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,oBAAoB,CAAC;CAC5D;AAkBD,qBAAa,cAAc;IACzB,OAAO,CAAC,EAAE,CAAuB;IACjC,OAAO,CAAC,WAAW,CAA+C;IAClE,OAAO,CAAC,MAAM,CAAS;gBAEX,IAAI,EAAE,qBAAqB;IA4BvC;;6DAEyD;IACzD,OAAO,CAAC,MAAM,CAAC,OAAO;IAItB;0DACsD;IACtD,oBAAoB,CAAC,CAAC,EAAE,mBAAmB,GAAG,IAAI;IAkBlD,kEAAkE;IAClE,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,mBAAmB;IA6BtE,0CAA0C;IAC1C,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,EAAE;IAapE,uCAAuC;IACvC,eAAe,CAAC,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,iBAAiB,EAAE;IAerF,KAAK,IAAI,IAAI;CAKd"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ResourceLedger — per-agent resource-usage observability (Phase A: rate-limit events).
|
|
3
|
+
*
|
|
4
|
+
* Durable, read-only. Mirrors TokenLedger / FeatureMetricsLedger: SQLite opened
|
|
5
|
+
* through NativeModuleHealer, WAL, registered for close-on-exit via
|
|
6
|
+
* SqliteRegistry, schema applied idempotently. Like FeatureMetricsLedger, every
|
|
7
|
+
* write swallows its error so observability can NEVER break the path it
|
|
8
|
+
* observes. It never gates, throttles, or mutates any runtime flow.
|
|
9
|
+
*
|
|
10
|
+
* Phase A persists the one resource signal that is ephemeral today: rate-limit
|
|
11
|
+
* events. LlmCircuitBreaker counts trips in process-local memory (lost on
|
|
12
|
+
* restart) and RateLimitSentinel emits transient events — so "how many times was
|
|
13
|
+
* the account throttled today" is currently unanswerable. This store makes it
|
|
14
|
+
* durable. Each event is written event-driven by ResourceLedgerPoller (from the
|
|
15
|
+
* breaker's trip/recover emitter and the sentinel's events), one row per
|
|
16
|
+
* emission, with a real per-event identity (not a content hash) so legitimate
|
|
17
|
+
* same-millisecond events don't collapse and a restart can't replay-double-count.
|
|
18
|
+
*
|
|
19
|
+
* CPU/memory sampling (resource_samples) + retention is Phase B; the table and
|
|
20
|
+
* its methods are not in this file yet.
|
|
21
|
+
*
|
|
22
|
+
* Spec: docs/specs/per-agent-resource-ledger.md.
|
|
23
|
+
*/
|
|
24
|
+
import * as fs from 'node:fs';
|
|
25
|
+
import * as path from 'node:path';
|
|
26
|
+
import Database from 'better-sqlite3';
|
|
27
|
+
import { registerSqliteHandle } from '../core/SqliteRegistry.js';
|
|
28
|
+
import { NativeModuleHealer } from '../memory/NativeModuleHealer.js';
|
|
29
|
+
const SCHEMA = [
|
|
30
|
+
`CREATE TABLE IF NOT EXISTS rate_limit_events (
|
|
31
|
+
id TEXT PRIMARY KEY,
|
|
32
|
+
ts INTEGER NOT NULL,
|
|
33
|
+
kind TEXT NOT NULL,
|
|
34
|
+
source TEXT NOT NULL,
|
|
35
|
+
account_key TEXT NOT NULL DEFAULT 'default',
|
|
36
|
+
session_name TEXT,
|
|
37
|
+
reason TEXT,
|
|
38
|
+
detail TEXT
|
|
39
|
+
)`,
|
|
40
|
+
`CREATE INDEX IF NOT EXISTS idx_rle_ts ON rate_limit_events (ts)`,
|
|
41
|
+
`CREATE INDEX IF NOT EXISTS idx_rle_kind_ts ON rate_limit_events (kind, ts)`,
|
|
42
|
+
`CREATE INDEX IF NOT EXISTS idx_rle_source_ts ON rate_limit_events (source, ts)`,
|
|
43
|
+
];
|
|
44
|
+
export class ResourceLedger {
|
|
45
|
+
db;
|
|
46
|
+
insertEvent;
|
|
47
|
+
closed = false;
|
|
48
|
+
constructor(opts) {
|
|
49
|
+
if (opts.dbPath !== ':memory:') {
|
|
50
|
+
fs.mkdirSync(path.dirname(opts.dbPath), { recursive: true });
|
|
51
|
+
}
|
|
52
|
+
// Open through the native-module healer — identical ABI-resilience to
|
|
53
|
+
// TokenLedger/FeatureMetricsLedger, so a Node upgrade can't brick /resources.
|
|
54
|
+
this.db = NativeModuleHealer.openWithHealSync('ResourceLedger', () => opts.databaseFactory?.(opts.dbPath) ?? new Database(opts.dbPath));
|
|
55
|
+
this.db.pragma('journal_mode = WAL');
|
|
56
|
+
this.db.pragma('synchronous = NORMAL');
|
|
57
|
+
for (const ddl of SCHEMA) {
|
|
58
|
+
try {
|
|
59
|
+
this.db.exec(ddl);
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
const msg = err.message || '';
|
|
63
|
+
if (!/duplicate column name/i.test(msg))
|
|
64
|
+
throw err;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
registerSqliteHandle(() => { try {
|
|
68
|
+
this.db?.close();
|
|
69
|
+
}
|
|
70
|
+
catch { /* already closed */ } });
|
|
71
|
+
this.insertEvent = this.db.prepare(`INSERT OR IGNORE INTO rate_limit_events
|
|
72
|
+
(id, ts, kind, source, account_key, session_name, reason, detail)
|
|
73
|
+
VALUES (@id, @ts, @kind, @source, @accountKey, @sessionName, @reason, @detail)`);
|
|
74
|
+
}
|
|
75
|
+
/** Per-event identity: ts + source + seq is unique within a process; the source
|
|
76
|
+
* prefix keeps breaker and sentinel rows from ever colliding. NOT a content
|
|
77
|
+
* hash, so two genuine same-ms events stay distinct. */
|
|
78
|
+
static eventId(e) {
|
|
79
|
+
return `${e.source}:${e.ts}:${e.seq}`;
|
|
80
|
+
}
|
|
81
|
+
/** Record one rate-limit event. Idempotent on (source,ts,seq). Never throws —
|
|
82
|
+
* observability must not break the observed path. */
|
|
83
|
+
recordRateLimitEvent(e) {
|
|
84
|
+
if (this.closed)
|
|
85
|
+
return;
|
|
86
|
+
try {
|
|
87
|
+
this.insertEvent.run({
|
|
88
|
+
id: ResourceLedger.eventId(e),
|
|
89
|
+
ts: e.ts,
|
|
90
|
+
kind: e.kind,
|
|
91
|
+
source: e.source,
|
|
92
|
+
accountKey: e.accountKey && e.accountKey.trim() ? e.accountKey.trim() : 'default',
|
|
93
|
+
sessionName: e.sessionName ?? null,
|
|
94
|
+
reason: e.reason ?? null,
|
|
95
|
+
detail: e.detail ?? null,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
/* swallow — never break the observed path (Close the Loop applied to self) */
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/** Count + rate of rate-limit events over the trailing window. */
|
|
103
|
+
rateLimitSummary(nowMs, windowMs) {
|
|
104
|
+
const since = nowMs - windowMs;
|
|
105
|
+
const row = this.db
|
|
106
|
+
.prepare(`SELECT
|
|
107
|
+
SUM(CASE WHEN kind='circuit-open' THEN 1 ELSE 0 END) AS circuitOpenCount,
|
|
108
|
+
SUM(CASE WHEN kind='circuit-recover' THEN 1 ELSE 0 END) AS circuitRecoverCount,
|
|
109
|
+
SUM(CASE WHEN source='session-sentinel' THEN 1 ELSE 0 END) AS sentinelCount,
|
|
110
|
+
COUNT(*) AS totalEvents,
|
|
111
|
+
MIN(ts) AS oldestEventTs,
|
|
112
|
+
MAX(ts) AS newestEventTs
|
|
113
|
+
FROM rate_limit_events
|
|
114
|
+
WHERE ts >= ?`)
|
|
115
|
+
.get(since);
|
|
116
|
+
const trips = Number(row.circuitOpenCount ?? 0);
|
|
117
|
+
const hours = windowMs / 3_600_000;
|
|
118
|
+
return {
|
|
119
|
+
circuitOpenCount: trips,
|
|
120
|
+
circuitRecoverCount: Number(row.circuitRecoverCount ?? 0),
|
|
121
|
+
sentinelCount: Number(row.sentinelCount ?? 0),
|
|
122
|
+
totalEvents: Number(row.totalEvents ?? 0),
|
|
123
|
+
tripsPerHour: hours > 0 ? +(trips / hours).toFixed(2) : 0,
|
|
124
|
+
oldestEventTs: row.oldestEventTs ?? null,
|
|
125
|
+
newestEventTs: row.newestEventTs ?? null,
|
|
126
|
+
windowMs,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** Per-kind breakdown over the window. */
|
|
130
|
+
rateLimitByKind(nowMs, windowMs) {
|
|
131
|
+
const since = nowMs - windowMs;
|
|
132
|
+
return this.db
|
|
133
|
+
.prepare(`SELECT kind, source, COUNT(*) AS count, MIN(ts) AS firstTs, MAX(ts) AS lastTs
|
|
134
|
+
FROM rate_limit_events
|
|
135
|
+
WHERE ts >= ?
|
|
136
|
+
GROUP BY kind, source
|
|
137
|
+
ORDER BY count DESC`)
|
|
138
|
+
.all(since);
|
|
139
|
+
}
|
|
140
|
+
/** Recent event rows, newest first. */
|
|
141
|
+
rateLimitEvents(opts = {}) {
|
|
142
|
+
const limit = Math.min(Math.max(opts.limit ?? 100, 1), 1000);
|
|
143
|
+
const since = opts.sinceMs ?? 0;
|
|
144
|
+
return this.db
|
|
145
|
+
.prepare(`SELECT id, ts, kind, source, account_key AS accountKey,
|
|
146
|
+
session_name AS sessionName, reason, detail
|
|
147
|
+
FROM rate_limit_events
|
|
148
|
+
WHERE ts >= ?
|
|
149
|
+
ORDER BY ts DESC
|
|
150
|
+
LIMIT ?`)
|
|
151
|
+
.all(since, limit);
|
|
152
|
+
}
|
|
153
|
+
close() {
|
|
154
|
+
if (this.closed)
|
|
155
|
+
return;
|
|
156
|
+
this.closed = true;
|
|
157
|
+
try {
|
|
158
|
+
this.db.close();
|
|
159
|
+
}
|
|
160
|
+
catch { /* ignore */ }
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=ResourceLedger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ResourceLedger.js","sourceRoot":"","sources":["../../src/monitoring/ResourceLedger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAmErE,MAAM,MAAM,GAAG;IACb;;;;;;;;;KASG;IACH,iEAAiE;IACjE,4EAA4E;IAC5E,gFAAgF;CACjF,CAAC;AAEF,MAAM,OAAO,cAAc;IACjB,EAAE,CAAuB;IACzB,WAAW,CAA+C;IAC1D,MAAM,GAAG,KAAK,CAAC;IAEvB,YAAY,IAA2B;QACrC,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAC/B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,sEAAsE;QACtE,8EAA8E;QAC9E,IAAI,CAAC,EAAE,GAAG,kBAAkB,CAAC,gBAAgB,CAC3C,gBAAgB,EAChB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CACvE,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAI,GAAa,CAAC,OAAO,IAAI,EAAE,CAAC;gBACzC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;YACrD,CAAC;QACH,CAAC;QACD,oBAAoB,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;YAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAChC;;sFAEgF,CACjF,CAAC;IACJ,CAAC;IAED;;6DAEyD;IACjD,MAAM,CAAC,OAAO,CAAC,CAAsB;QAC3C,OAAO,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IACxC,CAAC;IAED;0DACsD;IACtD,oBAAoB,CAAC,CAAsB;QACzC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC;YACH,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC;gBACnB,EAAE,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7B,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;gBACjF,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,IAAI;gBAClC,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,8EAA8E;QAChF,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,gBAAgB,CAAC,KAAa,EAAE,QAAgB;QAC9C,MAAM,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN;;;;;;;;uBAQe,CAChB;aACA,GAAG,CAAC,KAAK,CAAkC,CAAC;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;QACnC,OAAO;YACL,gBAAgB,EAAE,KAAK;YACvB,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,CAAC;YACzD,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,CAAC;YAC7C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;YACzC,YAAY,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YACxC,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YACxC,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,0CAA0C;IAC1C,eAAe,CAAC,KAAa,EAAE,QAAgB;QAC7C,MAAM,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;QAC/B,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CACN;;;;6BAIqB,CACtB;aACA,GAAG,CAAC,KAAK,CAAuB,CAAC;IACtC,CAAC;IAED,uCAAuC;IACvC,eAAe,CAAC,OAA6C,EAAE;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CACN;;;;;iBAKS,CACV;aACA,GAAG,CAAC,KAAK,EAAE,KAAK,CAAwB,CAAC;IAC9C,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACjD,CAAC;CACF"}
|