instar 1.3.905 → 1.3.907
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +140 -6
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +20 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/MessagingToneGate.d.ts.map +1 -1
- package/dist/core/MessagingToneGate.js +21 -0
- package/dist/core/MessagingToneGate.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts +22 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +70 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/ProactiveSwapMonitor.d.ts +4 -0
- package/dist/core/ProactiveSwapMonitor.d.ts.map +1 -1
- package/dist/core/ProactiveSwapMonitor.js +17 -8
- package/dist/core/ProactiveSwapMonitor.js.map +1 -1
- package/dist/core/QuotaAwareScheduler.d.ts +3 -1
- package/dist/core/QuotaAwareScheduler.d.ts.map +1 -1
- package/dist/core/QuotaAwareScheduler.js +5 -4
- package/dist/core/QuotaAwareScheduler.js.map +1 -1
- package/dist/core/SessionManager.d.ts +19 -0
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +33 -0
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/SessionPoolFailoverRunner.d.ts +94 -0
- package/dist/core/SessionPoolFailoverRunner.d.ts.map +1 -0
- package/dist/core/SessionPoolFailoverRunner.js +89 -0
- package/dist/core/SessionPoolFailoverRunner.js.map +1 -0
- package/dist/core/SessionPoolRolloutDriver.d.ts +74 -0
- package/dist/core/SessionPoolRolloutDriver.d.ts.map +1 -0
- package/dist/core/SessionPoolRolloutDriver.js +78 -0
- package/dist/core/SessionPoolRolloutDriver.js.map +1 -0
- package/dist/core/SessionRefresh.d.ts +3 -0
- package/dist/core/SessionRefresh.d.ts.map +1 -1
- package/dist/core/SessionRefresh.js +19 -0
- package/dist/core/SessionRefresh.js.map +1 -1
- package/dist/core/SwapAntiThrash.d.ts +1 -0
- package/dist/core/SwapAntiThrash.d.ts.map +1 -1
- package/dist/core/SwapAntiThrash.js +9 -5
- package/dist/core/SwapAntiThrash.js.map +1 -1
- package/dist/core/SwapLedger.d.ts +2 -0
- package/dist/core/SwapLedger.d.ts.map +1 -1
- package/dist/core/SwapLedger.js.map +1 -1
- package/dist/core/deferral-floor.d.ts +50 -0
- package/dist/core/deferral-floor.d.ts.map +1 -0
- package/dist/core/deferral-floor.js +169 -0
- package/dist/core/deferral-floor.js.map +1 -0
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +12 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/sessionPoolFailoverCheck.d.ts +67 -0
- package/dist/core/sessionPoolFailoverCheck.d.ts.map +1 -0
- package/dist/core/sessionPoolFailoverCheck.js +57 -0
- package/dist/core/sessionPoolFailoverCheck.js.map +1 -0
- package/dist/core/types.d.ts +33 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/messaging/slack/SlackAdapter.d.ts +3 -0
- package/dist/messaging/slack/SlackAdapter.d.ts.map +1 -1
- package/dist/messaging/slack/SlackAdapter.js +18 -0
- package/dist/messaging/slack/SlackAdapter.js.map +1 -1
- package/dist/monitoring/BlockerLifecycleService.d.ts.map +1 -1
- package/dist/monitoring/BlockerLifecycleService.js +11 -5
- package/dist/monitoring/BlockerLifecycleService.js.map +1 -1
- package/dist/monitoring/MissingLoginSessionDetector.d.ts +159 -0
- package/dist/monitoring/MissingLoginSessionDetector.d.ts.map +1 -0
- package/dist/monitoring/MissingLoginSessionDetector.js +166 -0
- package/dist/monitoring/MissingLoginSessionDetector.js.map +1 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.d.ts +144 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.d.ts.map +1 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.js +152 -0
- package/dist/monitoring/SingleMachineFailoverGapDetector.js.map +1 -0
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +40 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/monitoring/missingLoginSessionWiring.d.ts +96 -0
- package/dist/monitoring/missingLoginSessionWiring.d.ts.map +1 -0
- package/dist/monitoring/missingLoginSessionWiring.js +97 -0
- package/dist/monitoring/missingLoginSessionWiring.js.map +1 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.d.ts +56 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.d.ts.map +1 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.js +48 -0
- package/dist/monitoring/singleMachineFailoverGapWiring.js.map +1 -0
- package/dist/server/AgentServer.d.ts +2 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +2 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/CapabilityIndex.d.ts.map +1 -1
- package/dist/server/CapabilityIndex.js +30 -0
- package/dist/server/CapabilityIndex.js.map +1 -1
- package/dist/server/routes.d.ts +2 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +41 -3
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +65 -65
- package/upgrades/1.3.906.md +20 -0
- package/upgrades/1.3.907.md +30 -0
- package/upgrades/side-effects/proactive-default-account-swap.md +124 -0
- package/upgrades/side-effects/throughput-window-semantics.md +79 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MissingLoginSessionDetector — signal-only guard that surfaces the
|
|
3
|
+
* "a live session is running on an account whose login has gone missing" gap
|
|
4
|
+
* BEFORE the session dies silently.
|
|
5
|
+
*
|
|
6
|
+
* The failure it closes (2026-07-22, the justin-gmail silent auth-death): a live
|
|
7
|
+
* autonomous session was running on an account whose LOCAL LOGIN went missing
|
|
8
|
+
* (the subscription pool flagged `identityDrift.repairState ===
|
|
9
|
+
* 'owner-relogin-required'`, `actualAccountId === 'missing-local-login'`) while
|
|
10
|
+
* that account still had ~92% of its quota free. The session coasted on a cached
|
|
11
|
+
* credential, then — once that credential expired — every turn began failing on
|
|
12
|
+
* auth and the session went SILENT for ~2.5 hours. NOTHING alerted the operator;
|
|
13
|
+
* they had to notice the silence themselves and re-login by hand. The drift WAS
|
|
14
|
+
* detected by the pool, but it was never raised as an alert, and the reactive
|
|
15
|
+
* standby classifier only fires when the user sends a message (which, for hours,
|
|
16
|
+
* they did not).
|
|
17
|
+
*
|
|
18
|
+
* This detector makes that gap loud + PROACTIVE: when a live session is bound to
|
|
19
|
+
* an account whose login is missing, it raises ONE deduped attention item naming
|
|
20
|
+
* the account and that a re-login is needed. It is SIGNAL-ONLY — it never swaps
|
|
21
|
+
* accounts, never re-logins, never touches a session. It only tells the operator
|
|
22
|
+
* (and future-me) that a running session is about to wall on a missing login.
|
|
23
|
+
*
|
|
24
|
+
* Why it is distinct from the existing surfaces (not a duplicate):
|
|
25
|
+
* - The subscription pool DETECTS the drift, but raises no alert on it.
|
|
26
|
+
* - The honest-standby StuckSignatureClassifier can name an auth failure, but
|
|
27
|
+
* only reactively — when the user messages. A heads-down run with no inbound
|
|
28
|
+
* message falls through it (exactly what happened). This fires proactively.
|
|
29
|
+
* - It correlates two facts the others don't join: "login is gone" AND "a live
|
|
30
|
+
* session depends on that account right now" — drift alone is not urgent;
|
|
31
|
+
* drift under a live session is.
|
|
32
|
+
*
|
|
33
|
+
* Pure + injected: every environment read is a callback, so this module unit
|
|
34
|
+
* tests with zero real managers and no I/O.
|
|
35
|
+
*/
|
|
36
|
+
/** An account whose LOCAL login has gone missing (re-login required to recover). */
|
|
37
|
+
export interface MissingLoginAccount {
|
|
38
|
+
accountId: string;
|
|
39
|
+
/** The login "slot" (config home) the account serves from — how a session is matched to it. */
|
|
40
|
+
configHome: string;
|
|
41
|
+
}
|
|
42
|
+
/** A live session and the login slot (config home) it is currently running on. */
|
|
43
|
+
export interface LiveSessionBinding {
|
|
44
|
+
sessionName: string;
|
|
45
|
+
configHome: string;
|
|
46
|
+
/** Optional topic id, purely for the operator-facing message (never load-bearing). */
|
|
47
|
+
topicId?: number | null;
|
|
48
|
+
}
|
|
49
|
+
/** The attention item this detector emits (shape kept minimal + transport-agnostic). */
|
|
50
|
+
export interface MissingLoginAttention {
|
|
51
|
+
title: string;
|
|
52
|
+
body: string;
|
|
53
|
+
priority: 'high';
|
|
54
|
+
/** Stable per-episode key so repeated ticks coalesce to ONE item. */
|
|
55
|
+
dedupKey: string;
|
|
56
|
+
source: 'missing-login-session';
|
|
57
|
+
}
|
|
58
|
+
export interface MissingLoginSessionDetectorDeps {
|
|
59
|
+
/** Dark gate. When false the detector is a strict no-op (never reads, never raises). */
|
|
60
|
+
enabled: () => boolean;
|
|
61
|
+
/**
|
|
62
|
+
* Observe-only mode. When true the detector computes the verdict and audits a
|
|
63
|
+
* would-raise, but does NOT raise the attention item (the graduated-rollout
|
|
64
|
+
* dry-run rung).
|
|
65
|
+
*/
|
|
66
|
+
dryRun: () => boolean;
|
|
67
|
+
/** Accounts whose local login is currently missing (owner-relogin-required). */
|
|
68
|
+
getMissingLoginAccounts: () => MissingLoginAccount[];
|
|
69
|
+
/** The live sessions and which login slot (config home) each runs on. */
|
|
70
|
+
getLiveSessions: () => LiveSessionBinding[];
|
|
71
|
+
/** Raise a deduped attention item. Only called on a genuine gap when not in dryRun. */
|
|
72
|
+
raiseAttention: (item: MissingLoginAttention) => void;
|
|
73
|
+
/** Optional structured audit sink (every transition, including no-ops with a reason). */
|
|
74
|
+
audit?: (event: string, detail: Record<string, unknown>) => void;
|
|
75
|
+
}
|
|
76
|
+
/** One affected pairing: a missing-login account with the live sessions depending on it. */
|
|
77
|
+
export interface StrandedSession {
|
|
78
|
+
accountId: string;
|
|
79
|
+
sessionNames: string[];
|
|
80
|
+
}
|
|
81
|
+
export interface MissingLoginTickResult {
|
|
82
|
+
ran: boolean;
|
|
83
|
+
/** True when at least one live session is running on a missing-login account. */
|
|
84
|
+
gapDetected: boolean;
|
|
85
|
+
/** The affected account→sessions pairings (empty when no gap). */
|
|
86
|
+
stranded: StrandedSession[];
|
|
87
|
+
/** Whether an attention item was actually raised (false in dryRun even on a gap). */
|
|
88
|
+
raised: boolean;
|
|
89
|
+
}
|
|
90
|
+
/** The resolved, typed config the detector runs with. */
|
|
91
|
+
export interface MissingLoginSessionResolvedConfig {
|
|
92
|
+
/** Dev-agent gate: live on a development agent, dark on the fleet (unless set). */
|
|
93
|
+
enabled: boolean;
|
|
94
|
+
/** Dry-run first even on a dev agent: compute + audit, but do NOT raise. */
|
|
95
|
+
dryRun: boolean;
|
|
96
|
+
}
|
|
97
|
+
/** The raw config block shape (all optional — everything defaults). */
|
|
98
|
+
export interface MissingLoginSessionConfigBlock {
|
|
99
|
+
enabled?: boolean;
|
|
100
|
+
dryRun?: boolean;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Resolve `monitoring.missingLoginSession` against the dev-agent gate.
|
|
104
|
+
* `resolveEnabled` is the injected `resolveDevAgentGate(explicit, config)` result
|
|
105
|
+
* (kept as a param so this module stays free of a hard import — the server wiring
|
|
106
|
+
* passes the real gate). `dryRun` defaults TRUE (the graduated-rollout first rung).
|
|
107
|
+
*/
|
|
108
|
+
export declare function resolveMissingLoginSessionConfig(block: MissingLoginSessionConfigBlock | undefined, resolveEnabled: (explicit: boolean | undefined) => boolean): MissingLoginSessionResolvedConfig;
|
|
109
|
+
/** The guard-posture grade for `GET /guards`: dark ▸ dry-run ▸ live. */
|
|
110
|
+
export declare function guardStatusFor(cfg: MissingLoginSessionResolvedConfig): 'dark' | 'dry-run' | 'live';
|
|
111
|
+
/** The `status()` snapshot (the future `GET /pool/missing-login` body core). */
|
|
112
|
+
export interface MissingLoginSessionStatus {
|
|
113
|
+
enabled: boolean;
|
|
114
|
+
dryRun: boolean;
|
|
115
|
+
lastTickAt: string | null;
|
|
116
|
+
/** The affected account→sessions pairings seen on the LAST tick. */
|
|
117
|
+
stranded: StrandedSession[];
|
|
118
|
+
counters: {
|
|
119
|
+
ticks: number;
|
|
120
|
+
raises: number;
|
|
121
|
+
wouldRaise: number;
|
|
122
|
+
errors: number;
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* The pure detector. One `tick()` = one evaluation. It raises at most one
|
|
127
|
+
* (deduped) aggregate attention item per episode; the attention layer's own
|
|
128
|
+
* dedup collapses repeated ticks so a persistent gap never floods.
|
|
129
|
+
*/
|
|
130
|
+
export declare class MissingLoginSessionDetector {
|
|
131
|
+
private readonly deps;
|
|
132
|
+
private readonly now;
|
|
133
|
+
private ticks;
|
|
134
|
+
private raises;
|
|
135
|
+
private wouldRaise;
|
|
136
|
+
private errors;
|
|
137
|
+
private lastTickAtMs;
|
|
138
|
+
private lastStranded;
|
|
139
|
+
constructor(deps: MissingLoginSessionDetectorDeps, now?: () => number);
|
|
140
|
+
/**
|
|
141
|
+
* One evaluation. Rides an existing shared timer (no timer of its own). Fails
|
|
142
|
+
* toward silence: any internal error increments the error counter and emits
|
|
143
|
+
* nothing — the shared tick must never crash. A dark guard is a strict no-op.
|
|
144
|
+
*/
|
|
145
|
+
tick(): MissingLoginTickResult;
|
|
146
|
+
/** The read-surface snapshot (feeds the future GET route). */
|
|
147
|
+
status(): MissingLoginSessionStatus;
|
|
148
|
+
private audit;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Pure correlation: for each missing-login account, collect the live sessions
|
|
152
|
+
* bound to its login slot (config home). Only accounts with ≥1 dependent live
|
|
153
|
+
* session are stranded (drift alone, with no session on it, is not a gap).
|
|
154
|
+
*/
|
|
155
|
+
export declare function computeStranded(missing: MissingLoginAccount[], sessions: LiveSessionBinding[]): StrandedSession[];
|
|
156
|
+
/** Build the operator-facing attention item for a confirmed gap. Plain language, no jargon. */
|
|
157
|
+
export declare function buildAttention(stranded: StrandedSession[]): MissingLoginAttention;
|
|
158
|
+
export declare const MISSING_LOGIN_SESSION_DEDUP_KEY = "missing-login-session";
|
|
159
|
+
//# sourceMappingURL=MissingLoginSessionDetector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MissingLoginSessionDetector.d.ts","sourceRoot":"","sources":["../../src/monitoring/MissingLoginSessionDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,oFAAoF;AACpF,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA+F;IAC/F,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,wFAAwF;AACxF,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,uBAAuB,CAAC;CACjC;AAED,MAAM,WAAW,+BAA+B;IAC9C,wFAAwF;IACxF,OAAO,EAAE,MAAM,OAAO,CAAC;IACvB;;;;OAIG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,gFAAgF;IAChF,uBAAuB,EAAE,MAAM,mBAAmB,EAAE,CAAC;IACrD,yEAAyE;IACzE,eAAe,EAAE,MAAM,kBAAkB,EAAE,CAAC;IAC5C,uFAAuF;IACvF,cAAc,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,IAAI,CAAC;IACtD,yFAAyF;IACzF,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AAED,4FAA4F;AAC5F,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB;IACrC,GAAG,EAAE,OAAO,CAAC;IACb,iFAAiF;IACjF,WAAW,EAAE,OAAO,CAAC;IACrB,kEAAkE;IAClE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,qFAAqF;IACrF,MAAM,EAAE,OAAO,CAAC;CACjB;AAMD,yDAAyD;AACzD,MAAM,WAAW,iCAAiC;IAChD,mFAAmF;IACnF,OAAO,EAAE,OAAO,CAAC;IACjB,4EAA4E;IAC5E,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,uEAAuE;AACvE,MAAM,WAAW,8BAA8B;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAC9C,KAAK,EAAE,8BAA8B,GAAG,SAAS,EACjD,cAAc,EAAE,CAAC,QAAQ,EAAE,OAAO,GAAG,SAAS,KAAK,OAAO,GACzD,iCAAiC,CAMnC;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,GAAG,EAAE,iCAAiC,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,CAElG;AAED,gFAAgF;AAChF,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,oEAAoE;IACpE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACjF;AAED;;;;GAIG;AACH,qBAAa,2BAA2B;IASpC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,YAAY,CAAyB;gBAG1B,IAAI,EAAE,+BAA+B,EACrC,GAAG,GAAE,MAAM,MAAiB;IAG/C;;;;OAIG;IACH,IAAI,IAAI,sBAAsB;IAuC9B,8DAA8D;IAC9D,MAAM,IAAI,yBAAyB;IAUnC,OAAO,CAAC,KAAK;CAOd;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,mBAAmB,EAAE,EAC9B,QAAQ,EAAE,kBAAkB,EAAE,GAC7B,eAAe,EAAE,CAWnB;AAED,+FAA+F;AAC/F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,qBAAqB,CAiBjF;AAED,eAAO,MAAM,+BAA+B,0BAAY,CAAC"}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MissingLoginSessionDetector — signal-only guard that surfaces the
|
|
3
|
+
* "a live session is running on an account whose login has gone missing" gap
|
|
4
|
+
* BEFORE the session dies silently.
|
|
5
|
+
*
|
|
6
|
+
* The failure it closes (2026-07-22, the justin-gmail silent auth-death): a live
|
|
7
|
+
* autonomous session was running on an account whose LOCAL LOGIN went missing
|
|
8
|
+
* (the subscription pool flagged `identityDrift.repairState ===
|
|
9
|
+
* 'owner-relogin-required'`, `actualAccountId === 'missing-local-login'`) while
|
|
10
|
+
* that account still had ~92% of its quota free. The session coasted on a cached
|
|
11
|
+
* credential, then — once that credential expired — every turn began failing on
|
|
12
|
+
* auth and the session went SILENT for ~2.5 hours. NOTHING alerted the operator;
|
|
13
|
+
* they had to notice the silence themselves and re-login by hand. The drift WAS
|
|
14
|
+
* detected by the pool, but it was never raised as an alert, and the reactive
|
|
15
|
+
* standby classifier only fires when the user sends a message (which, for hours,
|
|
16
|
+
* they did not).
|
|
17
|
+
*
|
|
18
|
+
* This detector makes that gap loud + PROACTIVE: when a live session is bound to
|
|
19
|
+
* an account whose login is missing, it raises ONE deduped attention item naming
|
|
20
|
+
* the account and that a re-login is needed. It is SIGNAL-ONLY — it never swaps
|
|
21
|
+
* accounts, never re-logins, never touches a session. It only tells the operator
|
|
22
|
+
* (and future-me) that a running session is about to wall on a missing login.
|
|
23
|
+
*
|
|
24
|
+
* Why it is distinct from the existing surfaces (not a duplicate):
|
|
25
|
+
* - The subscription pool DETECTS the drift, but raises no alert on it.
|
|
26
|
+
* - The honest-standby StuckSignatureClassifier can name an auth failure, but
|
|
27
|
+
* only reactively — when the user messages. A heads-down run with no inbound
|
|
28
|
+
* message falls through it (exactly what happened). This fires proactively.
|
|
29
|
+
* - It correlates two facts the others don't join: "login is gone" AND "a live
|
|
30
|
+
* session depends on that account right now" — drift alone is not urgent;
|
|
31
|
+
* drift under a live session is.
|
|
32
|
+
*
|
|
33
|
+
* Pure + injected: every environment read is a callback, so this module unit
|
|
34
|
+
* tests with zero real managers and no I/O.
|
|
35
|
+
*/
|
|
36
|
+
const DEDUP_KEY = 'missing-login-session';
|
|
37
|
+
/**
|
|
38
|
+
* Resolve `monitoring.missingLoginSession` against the dev-agent gate.
|
|
39
|
+
* `resolveEnabled` is the injected `resolveDevAgentGate(explicit, config)` result
|
|
40
|
+
* (kept as a param so this module stays free of a hard import — the server wiring
|
|
41
|
+
* passes the real gate). `dryRun` defaults TRUE (the graduated-rollout first rung).
|
|
42
|
+
*/
|
|
43
|
+
export function resolveMissingLoginSessionConfig(block, resolveEnabled) {
|
|
44
|
+
const b = block ?? {};
|
|
45
|
+
return {
|
|
46
|
+
enabled: resolveEnabled(typeof b.enabled === 'boolean' ? b.enabled : undefined),
|
|
47
|
+
dryRun: typeof b.dryRun === 'boolean' ? b.dryRun : true,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/** The guard-posture grade for `GET /guards`: dark ▸ dry-run ▸ live. */
|
|
51
|
+
export function guardStatusFor(cfg) {
|
|
52
|
+
return cfg.enabled ? (cfg.dryRun ? 'dry-run' : 'live') : 'dark';
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* The pure detector. One `tick()` = one evaluation. It raises at most one
|
|
56
|
+
* (deduped) aggregate attention item per episode; the attention layer's own
|
|
57
|
+
* dedup collapses repeated ticks so a persistent gap never floods.
|
|
58
|
+
*/
|
|
59
|
+
export class MissingLoginSessionDetector {
|
|
60
|
+
deps;
|
|
61
|
+
now;
|
|
62
|
+
ticks = 0;
|
|
63
|
+
raises = 0;
|
|
64
|
+
wouldRaise = 0;
|
|
65
|
+
errors = 0;
|
|
66
|
+
lastTickAtMs = 0;
|
|
67
|
+
lastStranded = [];
|
|
68
|
+
constructor(deps, now = Date.now) {
|
|
69
|
+
this.deps = deps;
|
|
70
|
+
this.now = now;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* One evaluation. Rides an existing shared timer (no timer of its own). Fails
|
|
74
|
+
* toward silence: any internal error increments the error counter and emits
|
|
75
|
+
* nothing — the shared tick must never crash. A dark guard is a strict no-op.
|
|
76
|
+
*/
|
|
77
|
+
tick() {
|
|
78
|
+
if (!this.deps.enabled()) {
|
|
79
|
+
return { ran: false, gapDetected: false, stranded: [], raised: false };
|
|
80
|
+
}
|
|
81
|
+
this.ticks += 1;
|
|
82
|
+
this.lastTickAtMs = this.now();
|
|
83
|
+
try {
|
|
84
|
+
const missing = this.deps.getMissingLoginAccounts();
|
|
85
|
+
const sessions = this.deps.getLiveSessions();
|
|
86
|
+
const stranded = computeStranded(missing, sessions);
|
|
87
|
+
this.lastStranded = stranded;
|
|
88
|
+
if (stranded.length === 0) {
|
|
89
|
+
this.audit('no-gap', {
|
|
90
|
+
missingLoginAccounts: missing.length,
|
|
91
|
+
liveSessions: sessions.length,
|
|
92
|
+
});
|
|
93
|
+
return { ran: true, gapDetected: false, stranded: [], raised: false };
|
|
94
|
+
}
|
|
95
|
+
if (this.deps.dryRun()) {
|
|
96
|
+
this.wouldRaise += 1;
|
|
97
|
+
this.audit('would-raise', { stranded });
|
|
98
|
+
return { ran: true, gapDetected: true, stranded, raised: false };
|
|
99
|
+
}
|
|
100
|
+
this.deps.raiseAttention(buildAttention(stranded));
|
|
101
|
+
this.raises += 1;
|
|
102
|
+
this.audit('raised', { stranded });
|
|
103
|
+
return { ran: true, gapDetected: true, stranded, raised: true };
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
this.errors += 1;
|
|
107
|
+
this.audit('tick-error', { error: err instanceof Error ? err.message : String(err) });
|
|
108
|
+
return { ran: true, gapDetected: false, stranded: [], raised: false };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** The read-surface snapshot (feeds the future GET route). */
|
|
112
|
+
status() {
|
|
113
|
+
return {
|
|
114
|
+
enabled: this.deps.enabled(),
|
|
115
|
+
dryRun: this.deps.dryRun(),
|
|
116
|
+
lastTickAt: this.lastTickAtMs ? new Date(this.lastTickAtMs).toISOString() : null,
|
|
117
|
+
stranded: this.lastStranded,
|
|
118
|
+
counters: { ticks: this.ticks, raises: this.raises, wouldRaise: this.wouldRaise, errors: this.errors },
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
audit(event, detail) {
|
|
122
|
+
try {
|
|
123
|
+
this.deps.audit?.(event, detail);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
/* audit is best-effort — never let it break the tick */
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Pure correlation: for each missing-login account, collect the live sessions
|
|
132
|
+
* bound to its login slot (config home). Only accounts with ≥1 dependent live
|
|
133
|
+
* session are stranded (drift alone, with no session on it, is not a gap).
|
|
134
|
+
*/
|
|
135
|
+
export function computeStranded(missing, sessions) {
|
|
136
|
+
const out = [];
|
|
137
|
+
for (const acct of missing) {
|
|
138
|
+
const names = sessions
|
|
139
|
+
.filter((s) => s.configHome && s.configHome === acct.configHome)
|
|
140
|
+
.map((s) => s.sessionName);
|
|
141
|
+
if (names.length > 0) {
|
|
142
|
+
out.push({ accountId: acct.accountId, sessionNames: names });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
/** Build the operator-facing attention item for a confirmed gap. Plain language, no jargon. */
|
|
148
|
+
export function buildAttention(stranded) {
|
|
149
|
+
const totalSessions = stranded.reduce((n, s) => n + s.sessionNames.length, 0);
|
|
150
|
+
const sessionWord = totalSessions === 1 ? 'session' : 'sessions';
|
|
151
|
+
const acctList = stranded.map((s) => s.accountId).join(', ');
|
|
152
|
+
const acctWord = stranded.length === 1 ? 'account' : 'accounts';
|
|
153
|
+
const body = `${totalSessions} live ${sessionWord} ${totalSessions === 1 ? 'is' : 'are'} running on ${acctWord} ` +
|
|
154
|
+
`whose login has gone missing (${acctList}). The session works only until its cached credential ` +
|
|
155
|
+
`expires — then every turn fails on authentication and it goes silent with no error to you. ` +
|
|
156
|
+
`Re-login to that account now to keep it from going dark.`;
|
|
157
|
+
return {
|
|
158
|
+
title: 'A live session is running on a missing login',
|
|
159
|
+
body,
|
|
160
|
+
priority: 'high',
|
|
161
|
+
dedupKey: DEDUP_KEY,
|
|
162
|
+
source: 'missing-login-session',
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
export const MISSING_LOGIN_SESSION_DEDUP_KEY = DEDUP_KEY;
|
|
166
|
+
//# sourceMappingURL=MissingLoginSessionDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MissingLoginSessionDetector.js","sourceRoot":"","sources":["../../src/monitoring/MissingLoginSessionDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA8DH,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAkB1C;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,KAAiD,EACjD,cAA0D;IAE1D,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;IACtB,OAAO;QACL,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/E,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;KACxD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,GAAsC;IACnE,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAClE,CAAC;AAYD;;;;GAIG;AACH,MAAM,OAAO,2BAA2B;IASnB;IACA;IATX,KAAK,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,CAAC,CAAC;IACX,UAAU,GAAG,CAAC,CAAC;IACf,MAAM,GAAG,CAAC,CAAC;IACX,YAAY,GAAG,CAAC,CAAC;IACjB,YAAY,GAAsB,EAAE,CAAC;IAE7C,YACmB,IAAqC,EACrC,MAAoB,IAAI,CAAC,GAAG;QAD5B,SAAI,GAAJ,IAAI,CAAiC;QACrC,QAAG,GAAH,GAAG,CAAyB;IAC5C,CAAC;IAEJ;;;;OAIG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACzB,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAE7C,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;YAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE;oBACnB,oBAAoB,EAAE,OAAO,CAAC,MAAM;oBACpC,YAAY,EAAE,QAAQ,CAAC,MAAM;iBAC9B,CAAC,CAAC;gBACH,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACxE,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACxC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnE,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAClE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM;QACJ,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAC5B,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAC1B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;SACvG,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,KAAa,EAAE,MAA+B;QAC1D,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;IACH,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,OAA8B,EAC9B,QAA8B;IAE9B,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC;aAC/D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,cAAc,CAAC,QAA2B;IACxD,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,MAAM,WAAW,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;IACjE,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;IAChE,MAAM,IAAI,GACR,GAAG,aAAa,SAAS,WAAW,IAAI,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,eAAe,QAAQ,GAAG;QACpG,iCAAiC,QAAQ,wDAAwD;QACjG,6FAA6F;QAC7F,0DAA0D,CAAC;IAC7D,OAAO;QACL,KAAK,EAAE,8CAA8C;QACrD,IAAI;QACJ,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,SAAS;QACnB,MAAM,EAAE,uBAAuB;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,+BAA+B,GAAG,SAAS,CAAC"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SingleMachineFailoverGapDetector — signal-only guard that surfaces the
|
|
3
|
+
* "no failover target" gap BEFORE it bites.
|
|
4
|
+
*
|
|
5
|
+
* The failure it closes (2026-07-22, the Codey overnight loss): an agent was
|
|
6
|
+
* running autonomous work single-machine, its one machine's session stopped
|
|
7
|
+
* overnight, and NOTHING took over — because there was no second machine
|
|
8
|
+
* registered to fail over TO. The gap was invisible until a human noticed the
|
|
9
|
+
* next morning. Nothing structural surfaced "you have active autonomous work
|
|
10
|
+
* and no failover target".
|
|
11
|
+
*
|
|
12
|
+
* This detector makes that gap loud: when this agent is single-machine (no
|
|
13
|
+
* online mesh peer) WHILE it has active autonomous runs, it raises ONE deduped
|
|
14
|
+
* attention item. It is SIGNAL-ONLY — it never blocks, never provisions a peer,
|
|
15
|
+
* never touches a session. It only tells the operator (and future-me) that the
|
|
16
|
+
* running autonomous work has no takeover if this machine goes down.
|
|
17
|
+
*
|
|
18
|
+
* ── Honesty: two distinct gap modes ──
|
|
19
|
+
* not-configured : multiMachine is OFF entirely — no peer was ever registered.
|
|
20
|
+
* peer-offline : multiMachine is ON but every peer is currently offline —
|
|
21
|
+
* the failover target exists but is down right now.
|
|
22
|
+
* Both are a real gap when autonomous work is active (no takeover either way);
|
|
23
|
+
* the message names which so the operator knows whether to ADD a machine or
|
|
24
|
+
* REVIVE one.
|
|
25
|
+
*
|
|
26
|
+
* Pure + injected: every environment read is a callback, so this module unit
|
|
27
|
+
* tests with zero real managers and no I/O.
|
|
28
|
+
*/
|
|
29
|
+
export type FailoverGapMode = 'not-configured' | 'peer-offline';
|
|
30
|
+
/** A snapshot of this agent's mesh membership, as this machine sees it. */
|
|
31
|
+
export interface MeshMembership {
|
|
32
|
+
/** true iff multiMachine is configured/enabled on this agent at all. */
|
|
33
|
+
multiMachineEnabled: boolean;
|
|
34
|
+
/** Count of OTHER machines currently online in this agent's pool (excludes self). */
|
|
35
|
+
onlinePeerCount: number;
|
|
36
|
+
}
|
|
37
|
+
/** The attention item this detector emits (shape kept minimal + transport-agnostic). */
|
|
38
|
+
export interface FailoverGapAttention {
|
|
39
|
+
title: string;
|
|
40
|
+
body: string;
|
|
41
|
+
priority: 'high';
|
|
42
|
+
/** Stable per-episode key so repeated ticks coalesce to ONE item. */
|
|
43
|
+
dedupKey: string;
|
|
44
|
+
source: 'single-machine-failover-gap';
|
|
45
|
+
}
|
|
46
|
+
export interface SingleMachineFailoverGapDetectorDeps {
|
|
47
|
+
/** Dark gate. When false the detector is a strict no-op (never reads, never raises). */
|
|
48
|
+
enabled: () => boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Observe-only mode. When true the detector computes the verdict and audits a
|
|
51
|
+
* would-raise, but does NOT raise the attention item (the graduated-rollout
|
|
52
|
+
* dry-run rung).
|
|
53
|
+
*/
|
|
54
|
+
dryRun: () => boolean;
|
|
55
|
+
/** This agent's current mesh membership (single-machine ⇔ onlinePeerCount === 0). */
|
|
56
|
+
getMeshMembership: () => MeshMembership;
|
|
57
|
+
/** How many autonomous runs are active right now (the work that needs a failover target). */
|
|
58
|
+
getActiveAutonomousRunCount: () => number;
|
|
59
|
+
/** Raise a deduped attention item. Only called on a genuine gap when not in dryRun. */
|
|
60
|
+
raiseAttention: (item: FailoverGapAttention) => void;
|
|
61
|
+
/** Optional structured audit sink (every transition, including no-ops with a reason). */
|
|
62
|
+
audit?: (event: string, detail: Record<string, unknown>) => void;
|
|
63
|
+
}
|
|
64
|
+
export interface FailoverGapTickResult {
|
|
65
|
+
ran: boolean;
|
|
66
|
+
/** True when this agent is single-machine WITH active autonomous work. */
|
|
67
|
+
gapDetected: boolean;
|
|
68
|
+
/** Which mode of gap (only meaningful when gapDetected). */
|
|
69
|
+
mode: FailoverGapMode | null;
|
|
70
|
+
/** How many autonomous runs are exposed by the gap. */
|
|
71
|
+
atRiskRunCount: number;
|
|
72
|
+
/** Whether an attention item was actually raised (false in dryRun even on a gap). */
|
|
73
|
+
raised: boolean;
|
|
74
|
+
}
|
|
75
|
+
/** The resolved, typed config the detector runs with. */
|
|
76
|
+
export interface SingleMachineFailoverGapResolvedConfig {
|
|
77
|
+
/** Dev-agent gate: live on a development agent, dark on the fleet (unless set). */
|
|
78
|
+
enabled: boolean;
|
|
79
|
+
/** Dry-run first even on a dev agent: compute + audit, but do NOT raise. */
|
|
80
|
+
dryRun: boolean;
|
|
81
|
+
}
|
|
82
|
+
/** The raw config block shape (all optional — everything defaults). */
|
|
83
|
+
export interface SingleMachineFailoverGapConfigBlock {
|
|
84
|
+
enabled?: boolean;
|
|
85
|
+
dryRun?: boolean;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Resolve `monitoring.singleMachineFailoverGap` against the dev-agent gate.
|
|
89
|
+
* `resolveEnabled` is the injected `resolveDevAgentGate(explicit, config)` result
|
|
90
|
+
* (kept as a param so this module stays free of a hard import — the server wiring
|
|
91
|
+
* passes the real gate). `dryRun` defaults TRUE (the graduated-rollout first rung).
|
|
92
|
+
*/
|
|
93
|
+
export declare function resolveSingleMachineFailoverGapConfig(block: SingleMachineFailoverGapConfigBlock | undefined, resolveEnabled: (explicit: boolean | undefined) => boolean): SingleMachineFailoverGapResolvedConfig;
|
|
94
|
+
/** The guard-posture grade for `GET /guards`: dark ▸ dry-run ▸ live. */
|
|
95
|
+
export declare function guardStatusFor(cfg: SingleMachineFailoverGapResolvedConfig): 'dark' | 'dry-run' | 'live';
|
|
96
|
+
/** The `status()` snapshot (the future `GET /pool/failover-gap` body core). */
|
|
97
|
+
export interface SingleMachineFailoverGapStatus {
|
|
98
|
+
enabled: boolean;
|
|
99
|
+
dryRun: boolean;
|
|
100
|
+
lastTickAt: string | null;
|
|
101
|
+
/** True when the LAST tick saw this agent single-machine (no online peer). */
|
|
102
|
+
singleMachine: boolean;
|
|
103
|
+
/** Active autonomous runs seen on the LAST tick. */
|
|
104
|
+
activeAutonomousRunCount: number;
|
|
105
|
+
/** The current gap mode, or null when no gap is open. */
|
|
106
|
+
openGapMode: FailoverGapMode | null;
|
|
107
|
+
counters: {
|
|
108
|
+
ticks: number;
|
|
109
|
+
raises: number;
|
|
110
|
+
wouldRaise: number;
|
|
111
|
+
errors: number;
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The pure detector. One `tick()` = one evaluation. It raises at most one
|
|
116
|
+
* (deduped) attention item per gap episode; the attention layer's own dedup
|
|
117
|
+
* collapses repeated ticks so a persistent gap never floods.
|
|
118
|
+
*/
|
|
119
|
+
export declare class SingleMachineFailoverGapDetector {
|
|
120
|
+
private readonly deps;
|
|
121
|
+
private readonly now;
|
|
122
|
+
private ticks;
|
|
123
|
+
private raises;
|
|
124
|
+
private wouldRaise;
|
|
125
|
+
private errors;
|
|
126
|
+
private lastTickAtMs;
|
|
127
|
+
private lastSingleMachine;
|
|
128
|
+
private lastActiveRunCount;
|
|
129
|
+
private openGapMode;
|
|
130
|
+
constructor(deps: SingleMachineFailoverGapDetectorDeps, now?: () => number);
|
|
131
|
+
/**
|
|
132
|
+
* One evaluation. Rides an existing shared timer (no timer of its own). Fails
|
|
133
|
+
* toward silence: any internal error increments the error counter and emits
|
|
134
|
+
* nothing — the shared tick must never crash. A dark guard is a strict no-op.
|
|
135
|
+
*/
|
|
136
|
+
tick(): FailoverGapTickResult;
|
|
137
|
+
/** The read-surface snapshot (feeds the future GET route). */
|
|
138
|
+
status(): SingleMachineFailoverGapStatus;
|
|
139
|
+
private audit;
|
|
140
|
+
}
|
|
141
|
+
/** Build the operator-facing attention item for a confirmed gap. Plain language, no jargon. */
|
|
142
|
+
export declare function buildAttention(mode: FailoverGapMode, atRiskRunCount: number): FailoverGapAttention;
|
|
143
|
+
export declare const SINGLE_MACHINE_FAILOVER_GAP_DEDUP_KEY = "single-machine-failover-gap";
|
|
144
|
+
//# sourceMappingURL=SingleMachineFailoverGapDetector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SingleMachineFailoverGapDetector.d.ts","sourceRoot":"","sources":["../../src/monitoring/SingleMachineFailoverGapDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,MAAM,MAAM,eAAe,GAAG,gBAAgB,GAAG,cAAc,CAAC;AAEhE,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,mBAAmB,EAAE,OAAO,CAAC;IAC7B,qFAAqF;IACrF,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,wFAAwF;AACxF,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,6BAA6B,CAAC;CACvC;AAED,MAAM,WAAW,oCAAoC;IACnD,wFAAwF;IACxF,OAAO,EAAE,MAAM,OAAO,CAAC;IACvB;;;;OAIG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,qFAAqF;IACrF,iBAAiB,EAAE,MAAM,cAAc,CAAC;IACxC,6FAA6F;IAC7F,2BAA2B,EAAE,MAAM,MAAM,CAAC;IAC1C,uFAAuF;IACvF,cAAc,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACrD,yFAAyF;IACzF,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AAED,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,OAAO,CAAC;IACb,0EAA0E;IAC1E,WAAW,EAAE,OAAO,CAAC;IACrB,4DAA4D;IAC5D,IAAI,EAAE,eAAe,GAAG,IAAI,CAAC;IAC7B,uDAAuD;IACvD,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,MAAM,EAAE,OAAO,CAAC;CACjB;AAMD,yDAAyD;AACzD,MAAM,WAAW,sCAAsC;IACrD,mFAAmF;IACnF,OAAO,EAAE,OAAO,CAAC;IACjB,4EAA4E;IAC5E,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,uEAAuE;AACvE,MAAM,WAAW,mCAAmC;IAClD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;GAKG;AACH,wBAAgB,qCAAqC,CACnD,KAAK,EAAE,mCAAmC,GAAG,SAAS,EACtD,cAAc,EAAE,CAAC,QAAQ,EAAE,OAAO,GAAG,SAAS,KAAK,OAAO,GACzD,sCAAsC,CAMxC;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,GAAG,EAAE,sCAAsC,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,CAEvG;AAED,+EAA+E;AAC/E,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,8EAA8E;IAC9E,aAAa,EAAE,OAAO,CAAC;IACvB,oDAAoD;IACpD,wBAAwB,EAAE,MAAM,CAAC;IACjC,yDAAyD;IACzD,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;IACpC,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACjF;AAED;;;;GAIG;AACH,qBAAa,gCAAgC;IAWzC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAXtB,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,WAAW,CAAgC;gBAGhC,IAAI,EAAE,oCAAoC,EAC1C,GAAG,GAAE,MAAM,MAAiB;IAG/C;;;;OAIG;IACH,IAAI,IAAI,qBAAqB;IA8C7B,8DAA8D;IAC9D,MAAM,IAAI,8BAA8B;IAYxC,OAAO,CAAC,KAAK;CAOd;AAED,+FAA+F;AAC/F,wBAAgB,cAAc,CAAC,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,GAAG,oBAAoB,CAiBlG;AAED,eAAO,MAAM,qCAAqC,gCAAY,CAAC"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SingleMachineFailoverGapDetector — signal-only guard that surfaces the
|
|
3
|
+
* "no failover target" gap BEFORE it bites.
|
|
4
|
+
*
|
|
5
|
+
* The failure it closes (2026-07-22, the Codey overnight loss): an agent was
|
|
6
|
+
* running autonomous work single-machine, its one machine's session stopped
|
|
7
|
+
* overnight, and NOTHING took over — because there was no second machine
|
|
8
|
+
* registered to fail over TO. The gap was invisible until a human noticed the
|
|
9
|
+
* next morning. Nothing structural surfaced "you have active autonomous work
|
|
10
|
+
* and no failover target".
|
|
11
|
+
*
|
|
12
|
+
* This detector makes that gap loud: when this agent is single-machine (no
|
|
13
|
+
* online mesh peer) WHILE it has active autonomous runs, it raises ONE deduped
|
|
14
|
+
* attention item. It is SIGNAL-ONLY — it never blocks, never provisions a peer,
|
|
15
|
+
* never touches a session. It only tells the operator (and future-me) that the
|
|
16
|
+
* running autonomous work has no takeover if this machine goes down.
|
|
17
|
+
*
|
|
18
|
+
* ── Honesty: two distinct gap modes ──
|
|
19
|
+
* not-configured : multiMachine is OFF entirely — no peer was ever registered.
|
|
20
|
+
* peer-offline : multiMachine is ON but every peer is currently offline —
|
|
21
|
+
* the failover target exists but is down right now.
|
|
22
|
+
* Both are a real gap when autonomous work is active (no takeover either way);
|
|
23
|
+
* the message names which so the operator knows whether to ADD a machine or
|
|
24
|
+
* REVIVE one.
|
|
25
|
+
*
|
|
26
|
+
* Pure + injected: every environment read is a callback, so this module unit
|
|
27
|
+
* tests with zero real managers and no I/O.
|
|
28
|
+
*/
|
|
29
|
+
const DEDUP_KEY = 'single-machine-failover-gap';
|
|
30
|
+
/**
|
|
31
|
+
* Resolve `monitoring.singleMachineFailoverGap` against the dev-agent gate.
|
|
32
|
+
* `resolveEnabled` is the injected `resolveDevAgentGate(explicit, config)` result
|
|
33
|
+
* (kept as a param so this module stays free of a hard import — the server wiring
|
|
34
|
+
* passes the real gate). `dryRun` defaults TRUE (the graduated-rollout first rung).
|
|
35
|
+
*/
|
|
36
|
+
export function resolveSingleMachineFailoverGapConfig(block, resolveEnabled) {
|
|
37
|
+
const b = block ?? {};
|
|
38
|
+
return {
|
|
39
|
+
enabled: resolveEnabled(typeof b.enabled === 'boolean' ? b.enabled : undefined),
|
|
40
|
+
dryRun: typeof b.dryRun === 'boolean' ? b.dryRun : true,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/** The guard-posture grade for `GET /guards`: dark ▸ dry-run ▸ live. */
|
|
44
|
+
export function guardStatusFor(cfg) {
|
|
45
|
+
return cfg.enabled ? (cfg.dryRun ? 'dry-run' : 'live') : 'dark';
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The pure detector. One `tick()` = one evaluation. It raises at most one
|
|
49
|
+
* (deduped) attention item per gap episode; the attention layer's own dedup
|
|
50
|
+
* collapses repeated ticks so a persistent gap never floods.
|
|
51
|
+
*/
|
|
52
|
+
export class SingleMachineFailoverGapDetector {
|
|
53
|
+
deps;
|
|
54
|
+
now;
|
|
55
|
+
ticks = 0;
|
|
56
|
+
raises = 0;
|
|
57
|
+
wouldRaise = 0;
|
|
58
|
+
errors = 0;
|
|
59
|
+
lastTickAtMs = 0;
|
|
60
|
+
lastSingleMachine = false;
|
|
61
|
+
lastActiveRunCount = 0;
|
|
62
|
+
openGapMode = null;
|
|
63
|
+
constructor(deps, now = Date.now) {
|
|
64
|
+
this.deps = deps;
|
|
65
|
+
this.now = now;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* One evaluation. Rides an existing shared timer (no timer of its own). Fails
|
|
69
|
+
* toward silence: any internal error increments the error counter and emits
|
|
70
|
+
* nothing — the shared tick must never crash. A dark guard is a strict no-op.
|
|
71
|
+
*/
|
|
72
|
+
tick() {
|
|
73
|
+
if (!this.deps.enabled()) {
|
|
74
|
+
return { ran: false, gapDetected: false, mode: null, atRiskRunCount: 0, raised: false };
|
|
75
|
+
}
|
|
76
|
+
this.ticks += 1;
|
|
77
|
+
this.lastTickAtMs = this.now();
|
|
78
|
+
try {
|
|
79
|
+
const membership = this.deps.getMeshMembership();
|
|
80
|
+
const atRiskRunCount = this.deps.getActiveAutonomousRunCount();
|
|
81
|
+
this.lastSingleMachine = membership.onlinePeerCount <= 0;
|
|
82
|
+
this.lastActiveRunCount = atRiskRunCount;
|
|
83
|
+
const hasWorkNeedingFailover = atRiskRunCount > 0;
|
|
84
|
+
const gapDetected = this.lastSingleMachine && hasWorkNeedingFailover;
|
|
85
|
+
if (!gapDetected) {
|
|
86
|
+
this.openGapMode = null;
|
|
87
|
+
this.audit('no-gap', {
|
|
88
|
+
onlinePeerCount: membership.onlinePeerCount,
|
|
89
|
+
multiMachineEnabled: membership.multiMachineEnabled,
|
|
90
|
+
atRiskRunCount,
|
|
91
|
+
});
|
|
92
|
+
return { ran: true, gapDetected: false, mode: null, atRiskRunCount, raised: false };
|
|
93
|
+
}
|
|
94
|
+
const mode = membership.multiMachineEnabled ? 'peer-offline' : 'not-configured';
|
|
95
|
+
this.openGapMode = mode;
|
|
96
|
+
if (this.deps.dryRun()) {
|
|
97
|
+
this.wouldRaise += 1;
|
|
98
|
+
this.audit('would-raise', { mode, atRiskRunCount });
|
|
99
|
+
return { ran: true, gapDetected: true, mode, atRiskRunCount, raised: false };
|
|
100
|
+
}
|
|
101
|
+
this.deps.raiseAttention(buildAttention(mode, atRiskRunCount));
|
|
102
|
+
this.raises += 1;
|
|
103
|
+
this.audit('raised', { mode, atRiskRunCount });
|
|
104
|
+
return { ran: true, gapDetected: true, mode, atRiskRunCount, raised: true };
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
this.errors += 1;
|
|
108
|
+
this.audit('tick-error', { error: err instanceof Error ? err.message : String(err) });
|
|
109
|
+
return { ran: true, gapDetected: false, mode: null, atRiskRunCount: 0, raised: false };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/** The read-surface snapshot (feeds the future GET route). */
|
|
113
|
+
status() {
|
|
114
|
+
return {
|
|
115
|
+
enabled: this.deps.enabled(),
|
|
116
|
+
dryRun: this.deps.dryRun(),
|
|
117
|
+
lastTickAt: this.lastTickAtMs ? new Date(this.lastTickAtMs).toISOString() : null,
|
|
118
|
+
singleMachine: this.lastSingleMachine,
|
|
119
|
+
activeAutonomousRunCount: this.lastActiveRunCount,
|
|
120
|
+
openGapMode: this.openGapMode,
|
|
121
|
+
counters: { ticks: this.ticks, raises: this.raises, wouldRaise: this.wouldRaise, errors: this.errors },
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
audit(event, detail) {
|
|
125
|
+
try {
|
|
126
|
+
this.deps.audit?.(event, detail);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
/* audit is best-effort — never let it break the tick */
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/** Build the operator-facing attention item for a confirmed gap. Plain language, no jargon. */
|
|
134
|
+
export function buildAttention(mode, atRiskRunCount) {
|
|
135
|
+
const runWord = atRiskRunCount === 1 ? 'autonomous run' : 'autonomous runs';
|
|
136
|
+
const body = mode === 'not-configured'
|
|
137
|
+
? `${atRiskRunCount} ${runWord} are running on this machine with no second machine registered. ` +
|
|
138
|
+
`If this machine sleeps or the session stops, the work stops with nothing to take over. ` +
|
|
139
|
+
`Register a second machine so this agent has a failover target.`
|
|
140
|
+
: `${atRiskRunCount} ${runWord} are running, but every other machine in this agent's pool is ` +
|
|
141
|
+
`currently offline — so there is no failover target right now. Bring the other machine back ` +
|
|
142
|
+
`online, or the work has no takeover if this machine goes down.`;
|
|
143
|
+
return {
|
|
144
|
+
title: 'No failover target for active autonomous work',
|
|
145
|
+
body,
|
|
146
|
+
priority: 'high',
|
|
147
|
+
dedupKey: DEDUP_KEY,
|
|
148
|
+
source: 'single-machine-failover-gap',
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
export const SINGLE_MACHINE_FAILOVER_GAP_DEDUP_KEY = DEDUP_KEY;
|
|
152
|
+
//# sourceMappingURL=SingleMachineFailoverGapDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SingleMachineFailoverGapDetector.js","sourceRoot":"","sources":["../../src/monitoring/SingleMachineFailoverGapDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAqDH,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAkBhD;;;;;GAKG;AACH,MAAM,UAAU,qCAAqC,CACnD,KAAsD,EACtD,cAA0D;IAE1D,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;IACtB,OAAO;QACL,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/E,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;KACxD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,GAA2C;IACxE,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAClE,CAAC;AAgBD;;;;GAIG;AACH,MAAM,OAAO,gCAAgC;IAWxB;IACA;IAXX,KAAK,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,CAAC,CAAC;IACX,UAAU,GAAG,CAAC,CAAC;IACf,MAAM,GAAG,CAAC,CAAC;IACX,YAAY,GAAG,CAAC,CAAC;IACjB,iBAAiB,GAAG,KAAK,CAAC;IAC1B,kBAAkB,GAAG,CAAC,CAAC;IACvB,WAAW,GAA2B,IAAI,CAAC;IAEnD,YACmB,IAA0C,EAC1C,MAAoB,IAAI,CAAC,GAAG;QAD5B,SAAI,GAAJ,IAAI,CAAsC;QAC1C,QAAG,GAAH,GAAG,CAAyB;IAC5C,CAAC;IAEJ;;;;OAIG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACzB,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACjD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YAC/D,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,eAAe,IAAI,CAAC,CAAC;YACzD,IAAI,CAAC,kBAAkB,GAAG,cAAc,CAAC;YAEzC,MAAM,sBAAsB,GAAG,cAAc,GAAG,CAAC,CAAC;YAClD,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,sBAAsB,CAAC;YAErE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE;oBACnB,eAAe,EAAE,UAAU,CAAC,eAAe;oBAC3C,mBAAmB,EAAE,UAAU,CAAC,mBAAmB;oBACnD,cAAc;iBACf,CAAC,CAAC;gBACH,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACtF,CAAC;YAED,MAAM,IAAI,GAAoB,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,gBAAgB,CAAC;YACjG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAC/E,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;YAC/D,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;YAC/C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC9E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM;QACJ,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAC5B,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAC1B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAChF,aAAa,EAAE,IAAI,CAAC,iBAAiB;YACrC,wBAAwB,EAAE,IAAI,CAAC,kBAAkB;YACjD,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;SACvG,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,KAAa,EAAE,MAA+B;QAC1D,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;IACH,CAAC;CACF;AAED,+FAA+F;AAC/F,MAAM,UAAU,cAAc,CAAC,IAAqB,EAAE,cAAsB;IAC1E,MAAM,OAAO,GAAG,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC5E,MAAM,IAAI,GACR,IAAI,KAAK,gBAAgB;QACvB,CAAC,CAAC,GAAG,cAAc,IAAI,OAAO,kEAAkE;YAC9F,yFAAyF;YACzF,gEAAgE;QAClE,CAAC,CAAC,GAAG,cAAc,IAAI,OAAO,gEAAgE;YAC5F,6FAA6F;YAC7F,gEAAgE,CAAC;IACvE,OAAO;QACL,KAAK,EAAE,+CAA+C;QACtD,IAAI;QACJ,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,SAAS;QACnB,MAAM,EAAE,6BAA6B;KACtC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,qCAAqC,GAAG,SAAS,CAAC"}
|