4runr-os 2.10.72 → 2.10.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/gateway/package-lock.json +135 -120
- package/apps/gateway/scripts/verify-sentinel-persist.mjs +81 -81
- package/apps/gateway/src/__tests__/integration/sentinel.test.ts +79 -0
- package/apps/gateway/src/__tests__/sentinel-config-persist-http.test.ts +93 -93
- package/apps/gateway/src/__tests__/sentinel-config-store.test.ts +133 -133
- package/apps/gateway/src/security/sentinel-config-store.ts +208 -208
- package/dist/gateway-observability.d.ts +4 -0
- package/dist/gateway-observability.d.ts.map +1 -1
- package/dist/gateway-observability.js +14 -0
- package/dist/gateway-observability.js.map +1 -1
- package/dist/tui-handlers.js +222 -79
- package/dist/tui-handlers.js.map +1 -1
- package/dist/watchdog.d.ts +5 -0
- package/dist/watchdog.d.ts.map +1 -1
- package/dist/watchdog.js +82 -75
- package/dist/watchdog.js.map +1 -1
- package/mk3-tui/binaries/win32-x64/mk3-tui.exe +0 -0
- package/mk3-tui/src/app.rs +54 -0
- package/mk3-tui/src/main.rs +114 -0
- package/mk3-tui/src/ui/layout.rs +12 -0
- package/mk3-tui/src/ui/run_manager.rs +51 -6
- package/package.json +2 -2
- package/scripts/os-tools-smoke.cjs +548 -459
|
@@ -1,208 +1,208 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Persist operator Sentinel limits to disk (survives Gateway restart).
|
|
3
|
-
* File: {4Runr data dir}/config/sentinel-limits.json
|
|
4
|
-
*
|
|
5
|
-
* Boot order: env defaults (packages/sentinel) → overlay saved file when present.
|
|
6
|
-
* Explicit env vars for a field are not overwritten by the saved file.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import * as fs from 'fs';
|
|
10
|
-
import * as path from 'path';
|
|
11
|
-
import { get4RunrConfigDir } from '@4runr/shared';
|
|
12
|
-
import { Sentinel, type SentinelConfig } from '@4runr/sentinel';
|
|
13
|
-
|
|
14
|
-
const FILE_NAME = 'sentinel-limits.json';
|
|
15
|
-
|
|
16
|
-
const ENV_FIELD_KEYS: Array<{
|
|
17
|
-
field: keyof SentinelConfig;
|
|
18
|
-
envKeys: string[];
|
|
19
|
-
}> = [
|
|
20
|
-
{ field: 'enabled', envKeys: ['SENTINEL_ENABLED'] },
|
|
21
|
-
{ field: 'runMaxDurationMs', envKeys: ['RUN_MAX_DURATION_MS'] },
|
|
22
|
-
{ field: 'runMaxTokens', envKeys: ['RUN_MAX_TOKENS'] },
|
|
23
|
-
{ field: 'runIdleMs', envKeys: ['SENTINEL_IDLE_MS', 'RUN_IDLE_MS'] },
|
|
24
|
-
{ field: 'loopWindow', envKeys: ['SENTINEL_LOOP_WINDOW'] },
|
|
25
|
-
{ field: 'loopMax', envKeys: ['SENTINEL_LOOP_MAX'] },
|
|
26
|
-
{ field: 'runMaxCost', envKeys: ['RUN_MAX_COST', 'SENTINEL_MAX_COST'] },
|
|
27
|
-
];
|
|
28
|
-
|
|
29
|
-
export function getSentinelLimitsFilePath(): string {
|
|
30
|
-
return path.join(get4RunrConfigDir(), FILE_NAME);
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function envDefinesField(field: keyof SentinelConfig): boolean {
|
|
34
|
-
const entry = ENV_FIELD_KEYS.find((e) => e.field === field);
|
|
35
|
-
if (!entry) return false;
|
|
36
|
-
return entry.envKeys.some((k) => {
|
|
37
|
-
const v = process.env[k];
|
|
38
|
-
return v !== undefined && v.trim() !== '';
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
function parseSavedPayload(raw: unknown): SentinelConfig | null {
|
|
43
|
-
if (!raw || typeof raw !== 'object') return null;
|
|
44
|
-
const o = raw as Record<string, unknown>;
|
|
45
|
-
const num = (key: string): number | undefined => {
|
|
46
|
-
const v = o[key];
|
|
47
|
-
if (typeof v === 'number' && Number.isFinite(v)) return v;
|
|
48
|
-
return undefined;
|
|
49
|
-
};
|
|
50
|
-
const enabled = o['enabled'];
|
|
51
|
-
if (typeof enabled !== 'boolean') return null;
|
|
52
|
-
const runMaxDurationMs = num('runMaxDurationMs');
|
|
53
|
-
const runMaxTokens = num('runMaxTokens');
|
|
54
|
-
const runIdleMs = num('runIdleMs');
|
|
55
|
-
const loopWindow = num('loopWindow');
|
|
56
|
-
const loopMax = num('loopMax');
|
|
57
|
-
if (
|
|
58
|
-
runMaxDurationMs === undefined ||
|
|
59
|
-
runMaxTokens === undefined ||
|
|
60
|
-
runIdleMs === undefined ||
|
|
61
|
-
loopWindow === undefined ||
|
|
62
|
-
loopMax === undefined
|
|
63
|
-
) {
|
|
64
|
-
return null;
|
|
65
|
-
}
|
|
66
|
-
const runMaxCost = num('runMaxCost');
|
|
67
|
-
return {
|
|
68
|
-
enabled,
|
|
69
|
-
runMaxDurationMs,
|
|
70
|
-
runMaxTokens,
|
|
71
|
-
runIdleMs,
|
|
72
|
-
loopWindow,
|
|
73
|
-
loopMax,
|
|
74
|
-
...(runMaxCost !== undefined ? { runMaxCost } : {}),
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export function loadSentinelLimitsFromDisk(): SentinelConfig | null {
|
|
79
|
-
const filePath = getSentinelLimitsFilePath();
|
|
80
|
-
try {
|
|
81
|
-
if (!fs.existsSync(filePath)) return null;
|
|
82
|
-
const text = fs.readFileSync(filePath, 'utf8');
|
|
83
|
-
const parsed = JSON.parse(text) as unknown;
|
|
84
|
-
const inner =
|
|
85
|
-
parsed && typeof parsed === 'object' && 'config' in (parsed as object)
|
|
86
|
-
? (parsed as { config: unknown }).config
|
|
87
|
-
: parsed;
|
|
88
|
-
return parseSavedPayload(inner);
|
|
89
|
-
} catch {
|
|
90
|
-
return null;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
export function saveSentinelLimitsToDisk(config: SentinelConfig): void {
|
|
95
|
-
const filePath = getSentinelLimitsFilePath();
|
|
96
|
-
const payload = {
|
|
97
|
-
version: 1,
|
|
98
|
-
updatedAt: new Date().toISOString(),
|
|
99
|
-
config: {
|
|
100
|
-
enabled: config.enabled,
|
|
101
|
-
runMaxDurationMs: config.runMaxDurationMs,
|
|
102
|
-
runMaxTokens: config.runMaxTokens,
|
|
103
|
-
runIdleMs: config.runIdleMs,
|
|
104
|
-
loopWindow: config.loopWindow,
|
|
105
|
-
loopMax: config.loopMax,
|
|
106
|
-
runMaxCost: config.runMaxCost ?? 1.0,
|
|
107
|
-
},
|
|
108
|
-
};
|
|
109
|
-
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), { mode: 0o600 });
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
/** Merge saved limits from disk into the live Sentinel singleton (env wins per field). */
|
|
113
|
-
export function mergePersistedSentinelConfig(sentinel: Sentinel): boolean {
|
|
114
|
-
const saved = loadSentinelLimitsFromDisk();
|
|
115
|
-
if (!saved) return false;
|
|
116
|
-
|
|
117
|
-
const current = sentinel.getConfig();
|
|
118
|
-
const merged: SentinelConfig = { ...current };
|
|
119
|
-
|
|
120
|
-
for (const { field } of ENV_FIELD_KEYS) {
|
|
121
|
-
if (envDefinesField(field)) continue;
|
|
122
|
-
(merged as any)[field] = saved[field];
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
try {
|
|
126
|
-
sentinel.updateConfig(merged);
|
|
127
|
-
return true;
|
|
128
|
-
} catch {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Apply saved limits on Gateway boot. Env-defined fields keep env values.
|
|
135
|
-
*/
|
|
136
|
-
export function applyPersistedSentinelConfig(sentinel: Sentinel, logger?: {
|
|
137
|
-
info: (msg: string, meta?: Record<string, unknown>) => void;
|
|
138
|
-
warn: (msg: string, meta?: Record<string, unknown>) => void;
|
|
139
|
-
}): boolean {
|
|
140
|
-
const filePath = getSentinelLimitsFilePath();
|
|
141
|
-
const saved = loadSentinelLimitsFromDisk();
|
|
142
|
-
|
|
143
|
-
if (!saved) {
|
|
144
|
-
logger?.info('No Sentinel limits file found on disk (will use env defaults)', {
|
|
145
|
-
expectedPath: filePath,
|
|
146
|
-
});
|
|
147
|
-
return false;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// Check which env vars are set and will override the disk values
|
|
151
|
-
const envOverrides: string[] = [];
|
|
152
|
-
const diskApplied: string[] = [];
|
|
153
|
-
|
|
154
|
-
for (const { field, envKeys } of ENV_FIELD_KEYS) {
|
|
155
|
-
if (envDefinesField(field)) {
|
|
156
|
-
const activeEnv = envKeys.find(k => process.env[k] !== undefined && process.env[k]?.trim() !== '');
|
|
157
|
-
if (activeEnv) {
|
|
158
|
-
envOverrides.push(`${field} (via ${activeEnv}=${process.env[activeEnv]})`);
|
|
159
|
-
}
|
|
160
|
-
} else {
|
|
161
|
-
diskApplied.push(`${field}=${saved[field]}`);
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
const ok = mergePersistedSentinelConfig(sentinel);
|
|
166
|
-
|
|
167
|
-
if (ok) {
|
|
168
|
-
const finalConfig = sentinel.getConfig();
|
|
169
|
-
logger?.info('✅ Sentinel limits loaded from disk and merged', {
|
|
170
|
-
diskPath: filePath,
|
|
171
|
-
diskValues: saved,
|
|
172
|
-
envOverrides: envOverrides.length > 0 ? envOverrides : 'none',
|
|
173
|
-
diskAppliedFields: diskApplied.length > 0 ? diskApplied : 'none (all env-overridden)',
|
|
174
|
-
finalActiveConfig: finalConfig,
|
|
175
|
-
});
|
|
176
|
-
} else {
|
|
177
|
-
logger?.warn('Failed to apply Sentinel limits from disk', {
|
|
178
|
-
diskPath: filePath,
|
|
179
|
-
savedConfig: saved,
|
|
180
|
-
});
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
return ok;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/** Call before returning policy config to clients so long-lived Gateway picks up disk saves. */
|
|
187
|
-
export function hydrateSentinelConfigFromDisk(sentinel: Sentinel): void {
|
|
188
|
-
mergePersistedSentinelConfig(sentinel);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
export function persistSentinelConfigAfterApply(
|
|
192
|
-
sentinel: Sentinel,
|
|
193
|
-
config: SentinelConfig,
|
|
194
|
-
logger?: {
|
|
195
|
-
info: (msg: string, meta?: Record<string, unknown>) => void;
|
|
196
|
-
}
|
|
197
|
-
): string {
|
|
198
|
-
const filePath = getSentinelLimitsFilePath();
|
|
199
|
-
saveSentinelLimitsToDisk(config);
|
|
200
|
-
|
|
201
|
-
logger?.info('✅ Sentinel limits saved to disk', {
|
|
202
|
-
path: filePath,
|
|
203
|
-
config: config,
|
|
204
|
-
note: 'Survives Gateway restart. Env vars in docker-compose override individual fields when set.',
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
return filePath;
|
|
208
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Persist operator Sentinel limits to disk (survives Gateway restart).
|
|
3
|
+
* File: {4Runr data dir}/config/sentinel-limits.json
|
|
4
|
+
*
|
|
5
|
+
* Boot order: env defaults (packages/sentinel) → overlay saved file when present.
|
|
6
|
+
* Explicit env vars for a field are not overwritten by the saved file.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import * as fs from 'fs';
|
|
10
|
+
import * as path from 'path';
|
|
11
|
+
import { get4RunrConfigDir } from '@4runr/shared';
|
|
12
|
+
import { Sentinel, type SentinelConfig } from '@4runr/sentinel';
|
|
13
|
+
|
|
14
|
+
const FILE_NAME = 'sentinel-limits.json';
|
|
15
|
+
|
|
16
|
+
const ENV_FIELD_KEYS: Array<{
|
|
17
|
+
field: keyof SentinelConfig;
|
|
18
|
+
envKeys: string[];
|
|
19
|
+
}> = [
|
|
20
|
+
{ field: 'enabled', envKeys: ['SENTINEL_ENABLED'] },
|
|
21
|
+
{ field: 'runMaxDurationMs', envKeys: ['RUN_MAX_DURATION_MS'] },
|
|
22
|
+
{ field: 'runMaxTokens', envKeys: ['RUN_MAX_TOKENS'] },
|
|
23
|
+
{ field: 'runIdleMs', envKeys: ['SENTINEL_IDLE_MS', 'RUN_IDLE_MS'] },
|
|
24
|
+
{ field: 'loopWindow', envKeys: ['SENTINEL_LOOP_WINDOW'] },
|
|
25
|
+
{ field: 'loopMax', envKeys: ['SENTINEL_LOOP_MAX'] },
|
|
26
|
+
{ field: 'runMaxCost', envKeys: ['RUN_MAX_COST', 'SENTINEL_MAX_COST'] },
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
export function getSentinelLimitsFilePath(): string {
|
|
30
|
+
return path.join(get4RunrConfigDir(), FILE_NAME);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function envDefinesField(field: keyof SentinelConfig): boolean {
|
|
34
|
+
const entry = ENV_FIELD_KEYS.find((e) => e.field === field);
|
|
35
|
+
if (!entry) return false;
|
|
36
|
+
return entry.envKeys.some((k) => {
|
|
37
|
+
const v = process.env[k];
|
|
38
|
+
return v !== undefined && v.trim() !== '';
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function parseSavedPayload(raw: unknown): SentinelConfig | null {
|
|
43
|
+
if (!raw || typeof raw !== 'object') return null;
|
|
44
|
+
const o = raw as Record<string, unknown>;
|
|
45
|
+
const num = (key: string): number | undefined => {
|
|
46
|
+
const v = o[key];
|
|
47
|
+
if (typeof v === 'number' && Number.isFinite(v)) return v;
|
|
48
|
+
return undefined;
|
|
49
|
+
};
|
|
50
|
+
const enabled = o['enabled'];
|
|
51
|
+
if (typeof enabled !== 'boolean') return null;
|
|
52
|
+
const runMaxDurationMs = num('runMaxDurationMs');
|
|
53
|
+
const runMaxTokens = num('runMaxTokens');
|
|
54
|
+
const runIdleMs = num('runIdleMs');
|
|
55
|
+
const loopWindow = num('loopWindow');
|
|
56
|
+
const loopMax = num('loopMax');
|
|
57
|
+
if (
|
|
58
|
+
runMaxDurationMs === undefined ||
|
|
59
|
+
runMaxTokens === undefined ||
|
|
60
|
+
runIdleMs === undefined ||
|
|
61
|
+
loopWindow === undefined ||
|
|
62
|
+
loopMax === undefined
|
|
63
|
+
) {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
const runMaxCost = num('runMaxCost');
|
|
67
|
+
return {
|
|
68
|
+
enabled,
|
|
69
|
+
runMaxDurationMs,
|
|
70
|
+
runMaxTokens,
|
|
71
|
+
runIdleMs,
|
|
72
|
+
loopWindow,
|
|
73
|
+
loopMax,
|
|
74
|
+
...(runMaxCost !== undefined ? { runMaxCost } : {}),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function loadSentinelLimitsFromDisk(): SentinelConfig | null {
|
|
79
|
+
const filePath = getSentinelLimitsFilePath();
|
|
80
|
+
try {
|
|
81
|
+
if (!fs.existsSync(filePath)) return null;
|
|
82
|
+
const text = fs.readFileSync(filePath, 'utf8');
|
|
83
|
+
const parsed = JSON.parse(text) as unknown;
|
|
84
|
+
const inner =
|
|
85
|
+
parsed && typeof parsed === 'object' && 'config' in (parsed as object)
|
|
86
|
+
? (parsed as { config: unknown }).config
|
|
87
|
+
: parsed;
|
|
88
|
+
return parseSavedPayload(inner);
|
|
89
|
+
} catch {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function saveSentinelLimitsToDisk(config: SentinelConfig): void {
|
|
95
|
+
const filePath = getSentinelLimitsFilePath();
|
|
96
|
+
const payload = {
|
|
97
|
+
version: 1,
|
|
98
|
+
updatedAt: new Date().toISOString(),
|
|
99
|
+
config: {
|
|
100
|
+
enabled: config.enabled,
|
|
101
|
+
runMaxDurationMs: config.runMaxDurationMs,
|
|
102
|
+
runMaxTokens: config.runMaxTokens,
|
|
103
|
+
runIdleMs: config.runIdleMs,
|
|
104
|
+
loopWindow: config.loopWindow,
|
|
105
|
+
loopMax: config.loopMax,
|
|
106
|
+
runMaxCost: config.runMaxCost ?? 1.0,
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), { mode: 0o600 });
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Merge saved limits from disk into the live Sentinel singleton (env wins per field). */
|
|
113
|
+
export function mergePersistedSentinelConfig(sentinel: Sentinel): boolean {
|
|
114
|
+
const saved = loadSentinelLimitsFromDisk();
|
|
115
|
+
if (!saved) return false;
|
|
116
|
+
|
|
117
|
+
const current = sentinel.getConfig();
|
|
118
|
+
const merged: SentinelConfig = { ...current };
|
|
119
|
+
|
|
120
|
+
for (const { field } of ENV_FIELD_KEYS) {
|
|
121
|
+
if (envDefinesField(field)) continue;
|
|
122
|
+
(merged as any)[field] = saved[field];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
sentinel.updateConfig(merged);
|
|
127
|
+
return true;
|
|
128
|
+
} catch {
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Apply saved limits on Gateway boot. Env-defined fields keep env values.
|
|
135
|
+
*/
|
|
136
|
+
export function applyPersistedSentinelConfig(sentinel: Sentinel, logger?: {
|
|
137
|
+
info: (msg: string, meta?: Record<string, unknown>) => void;
|
|
138
|
+
warn: (msg: string, meta?: Record<string, unknown>) => void;
|
|
139
|
+
}): boolean {
|
|
140
|
+
const filePath = getSentinelLimitsFilePath();
|
|
141
|
+
const saved = loadSentinelLimitsFromDisk();
|
|
142
|
+
|
|
143
|
+
if (!saved) {
|
|
144
|
+
logger?.info('No Sentinel limits file found on disk (will use env defaults)', {
|
|
145
|
+
expectedPath: filePath,
|
|
146
|
+
});
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Check which env vars are set and will override the disk values
|
|
151
|
+
const envOverrides: string[] = [];
|
|
152
|
+
const diskApplied: string[] = [];
|
|
153
|
+
|
|
154
|
+
for (const { field, envKeys } of ENV_FIELD_KEYS) {
|
|
155
|
+
if (envDefinesField(field)) {
|
|
156
|
+
const activeEnv = envKeys.find(k => process.env[k] !== undefined && process.env[k]?.trim() !== '');
|
|
157
|
+
if (activeEnv) {
|
|
158
|
+
envOverrides.push(`${field} (via ${activeEnv}=${process.env[activeEnv]})`);
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
diskApplied.push(`${field}=${saved[field]}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const ok = mergePersistedSentinelConfig(sentinel);
|
|
166
|
+
|
|
167
|
+
if (ok) {
|
|
168
|
+
const finalConfig = sentinel.getConfig();
|
|
169
|
+
logger?.info('✅ Sentinel limits loaded from disk and merged', {
|
|
170
|
+
diskPath: filePath,
|
|
171
|
+
diskValues: saved,
|
|
172
|
+
envOverrides: envOverrides.length > 0 ? envOverrides : 'none',
|
|
173
|
+
diskAppliedFields: diskApplied.length > 0 ? diskApplied : 'none (all env-overridden)',
|
|
174
|
+
finalActiveConfig: finalConfig,
|
|
175
|
+
});
|
|
176
|
+
} else {
|
|
177
|
+
logger?.warn('Failed to apply Sentinel limits from disk', {
|
|
178
|
+
diskPath: filePath,
|
|
179
|
+
savedConfig: saved,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return ok;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Call before returning policy config to clients so long-lived Gateway picks up disk saves. */
|
|
187
|
+
export function hydrateSentinelConfigFromDisk(sentinel: Sentinel): void {
|
|
188
|
+
mergePersistedSentinelConfig(sentinel);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function persistSentinelConfigAfterApply(
|
|
192
|
+
sentinel: Sentinel,
|
|
193
|
+
config: SentinelConfig,
|
|
194
|
+
logger?: {
|
|
195
|
+
info: (msg: string, meta?: Record<string, unknown>) => void;
|
|
196
|
+
}
|
|
197
|
+
): string {
|
|
198
|
+
const filePath = getSentinelLimitsFilePath();
|
|
199
|
+
saveSentinelLimitsToDisk(config);
|
|
200
|
+
|
|
201
|
+
logger?.info('✅ Sentinel limits saved to disk', {
|
|
202
|
+
path: filePath,
|
|
203
|
+
config: config,
|
|
204
|
+
note: 'Survives Gateway restart. Env vars in docker-compose override individual fields when set.',
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
return filePath;
|
|
208
|
+
}
|
|
@@ -21,6 +21,10 @@ export declare function summarizeGatewayPrometheusMetrics(metricsText: string):
|
|
|
21
21
|
queueJobsWaiting: number;
|
|
22
22
|
queueJobsActive: number;
|
|
23
23
|
queueJobsFailed: number;
|
|
24
|
+
shieldDecisions: number;
|
|
25
|
+
shieldBlocks: number;
|
|
26
|
+
shieldMasks: number;
|
|
27
|
+
shieldRewrites: number;
|
|
24
28
|
};
|
|
25
29
|
topRoutes: Array<{
|
|
26
30
|
key: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway-observability.d.ts","sourceRoot":"","sources":["../src/gateway-observability.ts"],"names":[],"mappings":"AAAA;;GAEG;AAwDH;;GAEG;AACH,wBAAgB,iCAAiC,CAAC,WAAW,EAAE,MAAM,GAAG;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,gBAAgB,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"gateway-observability.d.ts","sourceRoot":"","sources":["../src/gateway-observability.ts"],"names":[],"mappings":"AAAA;;GAEG;AAwDH;;GAEG;AACH,wBAAgB,iCAAiC,CAAC,WAAW,EAAE,MAAM,GAAG;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,gBAAgB,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,SAAS,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB,CAoGA"}
|
|
@@ -78,6 +78,10 @@ export function summarizeGatewayPrometheusMetrics(metricsText) {
|
|
|
78
78
|
const queueJobsWaiting = firstGauge(m, 'queue_jobs_waiting');
|
|
79
79
|
const queueJobsActive = firstGauge(m, 'queue_jobs_active');
|
|
80
80
|
const queueJobsFailed = sumNamed(m, 'queue_jobs_failed_total');
|
|
81
|
+
const shieldDecisions = sumNamed(m, 'shield_decisions_total');
|
|
82
|
+
const shieldBlocks = sumNamed(m, 'shield_blocks_total');
|
|
83
|
+
const shieldMasks = sumNamed(m, 'shield_masks_total');
|
|
84
|
+
const shieldRewrites = sumNamed(m, 'shield_rewrites_total');
|
|
81
85
|
const durSum = sumNamed(m, 'http_request_duration_seconds_sum');
|
|
82
86
|
const durCount = sumNamed(m, 'http_request_duration_seconds_count');
|
|
83
87
|
let httpLatencyAvgMs = null;
|
|
@@ -110,6 +114,10 @@ export function summarizeGatewayPrometheusMetrics(metricsText) {
|
|
|
110
114
|
queueJobsWaiting,
|
|
111
115
|
queueJobsActive,
|
|
112
116
|
queueJobsFailed,
|
|
117
|
+
shieldDecisions,
|
|
118
|
+
shieldBlocks,
|
|
119
|
+
shieldMasks,
|
|
120
|
+
shieldRewrites,
|
|
113
121
|
};
|
|
114
122
|
const statsLines = [
|
|
115
123
|
`HTTP requests total ${httpRequests.toLocaleString()}`,
|
|
@@ -124,6 +132,12 @@ export function summarizeGatewayPrometheusMetrics(metricsText) {
|
|
|
124
132
|
`Queue jobs active ${queueJobsActive.toLocaleString()}`,
|
|
125
133
|
`Queue jobs failed ${queueJobsFailed.toLocaleString()}`,
|
|
126
134
|
`Idempotency keys (store) ${idempotencyStoreSize.toLocaleString()}`,
|
|
135
|
+
'',
|
|
136
|
+
'Shield (4Runr safety layer)',
|
|
137
|
+
`Shield decisions total ${shieldDecisions.toLocaleString()}`,
|
|
138
|
+
`Shield blocks total ${shieldBlocks.toLocaleString()}`,
|
|
139
|
+
`Shield masks total ${shieldMasks.toLocaleString()}`,
|
|
140
|
+
`Shield rewrites total ${shieldRewrites.toLocaleString()}`,
|
|
127
141
|
];
|
|
128
142
|
const routeLines = topRoutes.map((r) => {
|
|
129
143
|
const k = r.key.length > 36 ? `${r.key.slice(0, 33)}...` : r.key;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway-observability.js","sourceRoot":"","sources":["../src/gateway-observability.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,EAAE,cAAc,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;IAC3C,IAAI,CAAC,cAAc,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjF,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,WAAW,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAChE,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACjC,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAoE,CAAC;IACxF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1C,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,CAAwE,EAAE,IAAY;IACtG,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,CAAwE,EAAE,IAAY;IACxG,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,IAAI,EAAE,MAAM;QAAE,OAAO,CAAC,CAAC;IAC5B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,WAAmB;
|
|
1
|
+
{"version":3,"file":"gateway-observability.js","sourceRoot":"","sources":["../src/gateway-observability.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,EAAE,cAAc,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;IAC3C,IAAI,CAAC,cAAc,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACjF,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,WAAW,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAChE,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACjC,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAoE,CAAC;IACxF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1C,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,CAAwE,EAAE,IAAY;IACtG,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,CAAwE,EAAE,IAAY;IACxG,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,IAAI,EAAE,MAAM;QAAE,OAAO,CAAC,CAAC;IAC5B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,WAAmB;IA0BnE,MAAM,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;IAErC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,GAAG,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtF,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtD,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAC1D,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtD,MAAM,oBAAoB,GAAG,UAAU,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAErE,gBAAgB;IAChB,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,UAAU,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAE/D,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,EAAE,mCAAmC,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,EAAE,qCAAqC,CAAC,CAAC;IACpE,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAC3C,IAAI,QAAQ,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,WAAW,CAAC;QAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;QACxC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;SACvC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG;QACb,YAAY;QACZ,iBAAiB;QACjB,aAAa;QACb,WAAW;QACX,WAAW;QACX,aAAa;QACb,UAAU;QACV,SAAS;QACT,WAAW;QACX,oBAAoB;QACpB,gBAAgB;QAChB,gBAAgB;QAChB,eAAe;QACf,eAAe;QACf,eAAe;QACf,YAAY;QACZ,WAAW;QACX,cAAc;KACf,CAAC;IAEF,MAAM,UAAU,GAAG;QACjB,2BAA2B,YAAY,CAAC,cAAc,EAAE,EAAE;QAC1D,2BAA2B,iBAAiB,CAAC,cAAc,EAAE,KAAK,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAC9F,2BACE,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,cAAc,EAAE,KACxE,EAAE;QACF,2BAA2B,WAAW,CAAC,cAAc,EAAE,MAAM,WAAW,CAAC,cAAc,EAAE,EAAE;QAC3F,2BAA2B,aAAa,CAAC,cAAc,EAAE,EAAE;QAC3D,2BAA2B,UAAU,CAAC,cAAc,EAAE,EAAE;QACxD,2BAA2B,SAAS,CAAC,cAAc,EAAE,EAAE;QACvD,2BAA2B,WAAW,CAAC,cAAc,EAAE,EAAE;QACzD,2BAA2B,gBAAgB,CAAC,cAAc,EAAE,EAAE;QAC9D,2BAA2B,eAAe,CAAC,cAAc,EAAE,EAAE;QAC7D,2BAA2B,eAAe,CAAC,cAAc,EAAE,EAAE;QAC7D,4BAA4B,oBAAoB,CAAC,cAAc,EAAE,EAAE;QACnE,EAAE;QACF,6BAA6B;QAC7B,4BAA4B,eAAe,CAAC,cAAc,EAAE,EAAE;QAC9D,4BAA4B,YAAY,CAAC,cAAc,EAAE,EAAE;QAC3D,4BAA4B,WAAW,CAAC,cAAc,EAAE,EAAE;QAC1D,4BAA4B,cAAc,CAAC,cAAc,EAAE,EAAE;KAC9D,CAAC;IAEF,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACjE,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,MAAM;QACN,SAAS;QACT,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC"}
|