@tonyclaw/agent-inspector 3.1.20 → 3.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.output/backend/nitro.json +1 -1
- package/.output/cli.js +137 -20
- package/.output/server/_ssr/index.mjs +2 -2
- package/.output/server/_ssr/{router-DpXchCXm.mjs → router-DKFaWvAr.mjs} +2371 -1171
- package/.output/server/_tanstack-start-manifest_v-ClTaMkEj.mjs +4 -0
- package/.output/server/index.mjs +1 -1
- package/.output/ui/assets/{CompareDrawer-4i1QHaQf.js → CompareDrawer-pPcsyvc6.js} +1 -1
- package/.output/ui/assets/{InspectorPet-Cw6VJLr7.js → InspectorPet-DR3JEqze.js} +1 -1
- package/.output/ui/assets/ProxyViewerContainer-DOyYRnb-.js +60 -0
- package/.output/ui/assets/{ReplayDialog-DTs575iW.js → ReplayDialog-Z-1-gUDm.js} +1 -1
- package/.output/ui/assets/{RequestAnatomy-Cl6Es3gv.js → RequestAnatomy-BsnZjE5G.js} +1 -1
- package/.output/ui/assets/{ResponseView-DNCCLkzG.js → ResponseView-HY64_ZY2.js} +1 -1
- package/.output/ui/assets/{StreamingChunkSequence-DKaxRL9U.js → StreamingChunkSequence-CfQUFbTA.js} +1 -1
- package/.output/ui/assets/{_sessionId-rkR3WgRq.js → _sessionId-Dq97lD86.js} +1 -1
- package/.output/ui/assets/{_sessionId-C-YjWz50.js → _sessionId-_F9wwa3a.js} +1 -1
- package/.output/ui/assets/index-B-Qod-aA.css +1 -0
- package/.output/ui/assets/{index-DxiecWHw.js → index-B2yLwOEu.js} +1 -1
- package/.output/ui/assets/{index-D0d-V67x.js → index-CN-gZDqV.js} +1 -1
- package/.output/ui/assets/{index-DYbzZTDM.js → index-DOAGzrq2.js} +1 -1
- package/.output/ui/assets/{index-B48RD1-5.js → index-DYnqmBBx.js} +2 -2
- package/.output/ui/assets/{json-viewer-CjAwZhHA.js → json-viewer-B2SQN0Lg.js} +1 -1
- package/.output/ui/assets/{jszip.min-DEGEN1HP.js → jszip.min-Dpks67u2.js} +1 -1
- package/.output/ui/index.html +2 -2
- package/.output/workers/logFinalizer.worker.js +258 -0
- package/.output/workers/sessionWorkerEntry.js +258 -0
- package/package.json +13 -6
- package/src/backend/routes/api/logs.$id.replay.ts +6 -1
- package/src/backend/routes/api/logs.stream.ts +7 -0
- package/src/backend/routes/api/providers.$providerId.model-metadata.ts +15 -4
- package/src/backend/routes/api/storage.ts +56 -0
- package/src/backend/routes/metrics.ts +14 -0
- package/src/cli/doctor.ts +135 -2
- package/src/cli.ts +7 -4
- package/src/components/ProxyViewerContainer.tsx +41 -24
- package/src/components/providers/SettingsDialog.tsx +261 -70
- package/src/components/proxy-viewer/LogEntry.tsx +19 -5
- package/src/components/proxy-viewer/bodyHydration.ts +67 -0
- package/src/components/proxy-viewer/proxyViewerContainerLogic.ts +78 -0
- package/src/knowledge/openclawClient.ts +11 -2
- package/src/knowledge/openclawGatewayClient.ts +6 -1
- package/src/lib/resourceLimits.ts +191 -0
- package/src/lib/safeFetch.ts +428 -20
- package/src/proxy/ecosystemTasks.ts +20 -0
- package/src/proxy/handler.ts +17 -4
- package/src/proxy/identityProxy.ts +36 -16
- package/src/proxy/logIndex.ts +38 -6
- package/src/proxy/logger.ts +4 -0
- package/src/proxy/rawStreamCapture.ts +16 -1
- package/src/proxy/runtimeHealth.ts +4 -2
- package/src/proxy/runtimeMetrics.ts +149 -0
- package/src/proxy/sessionRuntime.ts +38 -0
- package/src/proxy/sqliteLogIndex.ts +51 -0
- package/src/proxy/storageLifecycle.ts +432 -0
- package/src/proxy/store.ts +62 -12
- package/.output/server/_tanstack-start-manifest_v-DnbdNeun.mjs +0 -4
- package/.output/ui/assets/ProxyViewerContainer-BST0B7lY.js +0 -60
- package/.output/ui/assets/index-IBEDJoV_.css +0 -1
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
signalProcessTree,
|
|
15
15
|
type CommandSpec,
|
|
16
16
|
} from "./platformCommands";
|
|
17
|
+
import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
|
|
17
18
|
|
|
18
19
|
const MAX_TASKS = 50;
|
|
19
20
|
const OUTPUT_LIMIT = 40_000;
|
|
@@ -34,6 +35,12 @@ const activeTasks = new Map<string, ActiveTask>();
|
|
|
34
35
|
let acceptingTasks = true;
|
|
35
36
|
let shutdownForceTimer: ReturnType<typeof setTimeout> | null = null;
|
|
36
37
|
|
|
38
|
+
function activeTaskLimit(): number {
|
|
39
|
+
const configured = Number(process.env["ECOSYSTEM_TASK_ACTIVE_LIMIT"]);
|
|
40
|
+
if (Number.isSafeInteger(configured) && configured > 0) return configured;
|
|
41
|
+
return DEFAULT_RUNTIME_BUDGET_POLICY.queues.backgroundWorkers;
|
|
42
|
+
}
|
|
43
|
+
|
|
37
44
|
function nowIso(): string {
|
|
38
45
|
return new Date().toISOString();
|
|
39
46
|
}
|
|
@@ -258,6 +265,7 @@ export function startEcosystemTask(
|
|
|
258
265
|
action: EcosystemTaskAction,
|
|
259
266
|
): EcosystemTask | null {
|
|
260
267
|
if (!acceptingTasks) return null;
|
|
268
|
+
if (activeTasks.size >= activeTaskLimit()) return null;
|
|
261
269
|
if (action === "recipe") return null;
|
|
262
270
|
const definition =
|
|
263
271
|
action === "runner-presets"
|
|
@@ -291,6 +299,7 @@ export function startEcosystemTask(
|
|
|
291
299
|
|
|
292
300
|
export function startEcosystemRecipeTask(recipeId: string): EcosystemTask | null {
|
|
293
301
|
if (!acceptingTasks) return null;
|
|
302
|
+
if (activeTasks.size >= activeTaskLimit()) return null;
|
|
294
303
|
const recipe = findEcosystemRecipe(recipeId);
|
|
295
304
|
if (recipe === null) return null;
|
|
296
305
|
if (!recipe.runnable) return null;
|
|
@@ -367,6 +376,7 @@ export function _startEcosystemTaskForTests(
|
|
|
367
376
|
timeoutMs = RECIPE_TIMEOUT_MS,
|
|
368
377
|
): EcosystemTask | null {
|
|
369
378
|
if (!acceptingTasks) return null;
|
|
379
|
+
if (activeTasks.size >= activeTaskLimit()) return null;
|
|
370
380
|
const task: EcosystemTask = {
|
|
371
381
|
id: randomUUID(),
|
|
372
382
|
packageId: "test-package",
|
|
@@ -393,3 +403,13 @@ export function _resetEcosystemTasksForTests(): void {
|
|
|
393
403
|
activeTasks.clear();
|
|
394
404
|
acceptingTasks = true;
|
|
395
405
|
}
|
|
406
|
+
|
|
407
|
+
export function _getEcosystemTaskAdmissionHealthForTests(): {
|
|
408
|
+
activeTasks: number;
|
|
409
|
+
activeTaskLimit: number;
|
|
410
|
+
} {
|
|
411
|
+
return {
|
|
412
|
+
activeTasks: activeTasks.size,
|
|
413
|
+
activeTaskLimit: activeTaskLimit(),
|
|
414
|
+
};
|
|
415
|
+
}
|
package/src/proxy/handler.ts
CHANGED
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
getRuntimeAdmissionState,
|
|
41
41
|
type RuntimeRequestLease,
|
|
42
42
|
} from "./runtimeAdmission";
|
|
43
|
-
import { resourceLimitResponse } from "../lib/resourceLimits";
|
|
43
|
+
import { DEFAULT_RUNTIME_BUDGET_POLICY, resourceLimitResponse } from "../lib/resourceLimits";
|
|
44
44
|
import {
|
|
45
45
|
buildUpstreamUrl,
|
|
46
46
|
describeApiRoute,
|
|
@@ -592,10 +592,19 @@ async function handleAdmittedProxy(req: Request): Promise<Response> {
|
|
|
592
592
|
body: bodyToForward,
|
|
593
593
|
signal: req.signal,
|
|
594
594
|
},
|
|
595
|
-
{
|
|
595
|
+
{
|
|
596
|
+
allowLoopback: ALLOW_LOOPBACK,
|
|
597
|
+
deadlineMs: DEFAULT_RUNTIME_BUDGET_POLICY.upstream.fetchDeadlineMs,
|
|
598
|
+
responseBytes: DEFAULT_RUNTIME_BUDGET_POLICY.upstream.fetchResponseBytes,
|
|
599
|
+
},
|
|
596
600
|
);
|
|
597
601
|
if (!fetchResult.ok) {
|
|
598
|
-
const status =
|
|
602
|
+
const status =
|
|
603
|
+
fetchResult.kind === "network-policy"
|
|
604
|
+
? STATUS_FORBIDDEN
|
|
605
|
+
: fetchResult.kind === "resource-limit"
|
|
606
|
+
? STATUS_PAYLOAD_TOO_LARGE
|
|
607
|
+
: STATUS_BAD_GATEWAY;
|
|
599
608
|
logger.warn(`[handler] Safe fetch blocked ${upstreamUrl}: ${fetchResult.reason}`);
|
|
600
609
|
log.elapsedMs = Date.now() - startTime;
|
|
601
610
|
log.responseStatus = status;
|
|
@@ -605,7 +614,11 @@ async function handleAdmittedProxy(req: Request): Promise<Response> {
|
|
|
605
614
|
return new Response(
|
|
606
615
|
fetchResult.kind === "network-policy"
|
|
607
616
|
? "Forbidden: upstream URL is not permitted"
|
|
608
|
-
:
|
|
617
|
+
: fetchResult.kind === "resource-limit"
|
|
618
|
+
? "Proxy error: upstream response exceeded configured limits"
|
|
619
|
+
: fetchResult.kind === "timeout"
|
|
620
|
+
? "Proxy error: upstream request timed out"
|
|
621
|
+
: "Proxy error: upstream redirects are not permitted",
|
|
609
622
|
{ status },
|
|
610
623
|
);
|
|
611
624
|
}
|
|
@@ -585,9 +585,10 @@ function normalizedHostname(value: string): string | null {
|
|
|
585
585
|
}
|
|
586
586
|
|
|
587
587
|
function isLoopbackHostname(hostname: string): boolean {
|
|
588
|
-
|
|
589
|
-
if (
|
|
590
|
-
|
|
588
|
+
const normalized = hostname.toLowerCase().replace(/^\[|]$/g, "");
|
|
589
|
+
if (normalized === "localhost" || normalized === "::1") return true;
|
|
590
|
+
if (isIP(normalized) !== 4) return false;
|
|
591
|
+
return normalized.split(".", 1)[0] === "127";
|
|
591
592
|
}
|
|
592
593
|
|
|
593
594
|
export type UiOriginAllowlist = {
|
|
@@ -669,15 +670,31 @@ function normalizedRequestOrigin(origin: string): string | null {
|
|
|
669
670
|
}
|
|
670
671
|
}
|
|
671
672
|
|
|
673
|
+
function parsedRequestOrigin(origin: string): URL | null {
|
|
674
|
+
const normalized = normalizedRequestOrigin(origin);
|
|
675
|
+
if (normalized === null) return null;
|
|
676
|
+
return new URL(normalized);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
function isSameOrLoopbackAliasOrigin(left: URL, right: URL): boolean {
|
|
680
|
+
if (left.origin === right.origin) return true;
|
|
681
|
+
return (
|
|
682
|
+
left.protocol === right.protocol &&
|
|
683
|
+
left.port === right.port &&
|
|
684
|
+
isLoopbackHostname(left.hostname) &&
|
|
685
|
+
isLoopbackHostname(right.hostname)
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
|
|
672
689
|
function isSameRequestOrigin(
|
|
673
690
|
origin: string,
|
|
674
691
|
host: string | undefined,
|
|
675
692
|
publicOrigin?: string,
|
|
676
693
|
): boolean {
|
|
677
|
-
const parsedOrigin =
|
|
694
|
+
const parsedOrigin = parsedRequestOrigin(origin);
|
|
678
695
|
if (parsedOrigin === null) return false;
|
|
679
|
-
const expected =
|
|
680
|
-
return expected !== null && parsedOrigin
|
|
696
|
+
const expected = parsedRequestOrigin(publicOrigin ?? `http://${host ?? "localhost"}`);
|
|
697
|
+
return expected !== null && isSameOrLoopbackAliasOrigin(parsedOrigin, expected);
|
|
681
698
|
}
|
|
682
699
|
|
|
683
700
|
function matchedUiOrigin(
|
|
@@ -797,24 +814,27 @@ export function isBrowserMutationAllowed(input: {
|
|
|
797
814
|
const fetchSite = input.secFetchSite?.toLowerCase();
|
|
798
815
|
const browserRequest = input.origin !== undefined || fetchSite !== undefined;
|
|
799
816
|
if (!browserRequest) return true;
|
|
817
|
+
const sameOrigin =
|
|
818
|
+
input.origin !== undefined
|
|
819
|
+
? isSameRequestOrigin(input.origin, input.host, input.publicOrigin)
|
|
820
|
+
: false;
|
|
800
821
|
if (isModelProxyPath(pathname)) {
|
|
801
|
-
if (fetchSite !== undefined && fetchSite !== "same-origin") return false;
|
|
802
|
-
if (
|
|
803
|
-
input.origin !== undefined &&
|
|
804
|
-
!isSameRequestOrigin(input.origin, input.host, input.publicOrigin)
|
|
805
|
-
) {
|
|
822
|
+
if (fetchSite !== undefined && fetchSite !== "same-origin" && !sameOrigin) return false;
|
|
823
|
+
if (input.origin !== undefined && !sameOrigin) {
|
|
806
824
|
return false;
|
|
807
825
|
}
|
|
808
826
|
return input.csrfToken === "1";
|
|
809
827
|
}
|
|
810
828
|
const allowedOrigin = matchedUiOrigin(input.origin, input.allowedUiOrigins);
|
|
811
|
-
if (
|
|
829
|
+
if (
|
|
830
|
+
fetchSite !== undefined &&
|
|
831
|
+
fetchSite !== "same-origin" &&
|
|
832
|
+
!sameOrigin &&
|
|
833
|
+
allowedOrigin === null
|
|
834
|
+
)
|
|
812
835
|
return false;
|
|
813
836
|
if (input.origin !== undefined) {
|
|
814
|
-
if (
|
|
815
|
-
!isSameRequestOrigin(input.origin, input.host, input.publicOrigin) &&
|
|
816
|
-
allowedOrigin === null
|
|
817
|
-
) {
|
|
837
|
+
if (!sameOrigin && allowedOrigin === null) {
|
|
818
838
|
return false;
|
|
819
839
|
}
|
|
820
840
|
}
|
package/src/proxy/logIndex.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { extractToolTraceEvents } from "../lib/toolTrace";
|
|
|
17
17
|
import { extractStopReason } from "../lib/stopReason";
|
|
18
18
|
import {
|
|
19
19
|
getSqliteLogIndexMaxId,
|
|
20
|
+
listSqliteLogCursorEntries,
|
|
20
21
|
replaceSqliteLogIndexEntries,
|
|
21
22
|
syncSqliteLogIndexEntries,
|
|
22
23
|
upsertSqliteLogIndexEntry,
|
|
@@ -159,9 +160,36 @@ function createEmptyIndex(): LogIndex {
|
|
|
159
160
|
};
|
|
160
161
|
}
|
|
161
162
|
|
|
163
|
+
function createIndexFromEntries(entries: readonly LogIndexEntry[]): LogIndex {
|
|
164
|
+
const index = createEmptyIndex();
|
|
165
|
+
for (const entry of entries) {
|
|
166
|
+
index.entries[entry.id] = entry;
|
|
167
|
+
index.maxId = Math.max(index.maxId, entry.id);
|
|
168
|
+
}
|
|
169
|
+
return index;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function sqliteProjectionReferencesExistingLedgers(entries: readonly LogIndexEntry[]): boolean {
|
|
173
|
+
const logDir = resolveLogDir();
|
|
174
|
+
for (const entry of entries) {
|
|
175
|
+
if (!entry.file.endsWith(".jsonl")) return false;
|
|
176
|
+
if (!existsSync(join(logDir, entry.file))) return false;
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
|
|
162
181
|
export async function loadIndex(): Promise<LogIndex> {
|
|
163
182
|
if (cachedIndex !== null) return cachedIndex;
|
|
164
|
-
|
|
183
|
+
const sqliteEntries = await listSqliteLogCursorEntries({});
|
|
184
|
+
if (sqliteEntries !== null) {
|
|
185
|
+
if (sqliteProjectionReferencesExistingLedgers(sqliteEntries)) {
|
|
186
|
+
cachedIndex = createIndexFromEntries(sqliteEntries);
|
|
187
|
+
allocatedMaxId = Math.max(allocatedMaxId, cachedIndex.maxId);
|
|
188
|
+
return cachedIndex;
|
|
189
|
+
}
|
|
190
|
+
logger.warn("[logIndex] SQLite projection references missing ledgers; rebuilding from JSONL");
|
|
191
|
+
}
|
|
192
|
+
// logs.idx remains a legacy fallback projection only. If SQLite is unavailable, rebuild from
|
|
165
193
|
// authoritative JSONL rather than allowing a stale, conflicting, or higher derived row to win.
|
|
166
194
|
return await rebuildIndex();
|
|
167
195
|
}
|
|
@@ -212,9 +240,11 @@ export async function addToIndex(
|
|
|
212
240
|
if (id > index.maxId) {
|
|
213
241
|
index.maxId = id;
|
|
214
242
|
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
243
|
+
const sqliteUpdated = await upsertSqliteLogIndexEntry(entry);
|
|
244
|
+
if (!sqliteUpdated) {
|
|
245
|
+
// Legacy fallback only: defer disk writes to reduce I/O when SQLite is unavailable.
|
|
246
|
+
scheduleIndexFlush();
|
|
247
|
+
}
|
|
218
248
|
}
|
|
219
249
|
|
|
220
250
|
/**
|
|
@@ -232,8 +262,8 @@ export async function updateLogIndexEntryMetadata(log: CapturedLog): Promise<voi
|
|
|
232
262
|
...createLogIndexEntryMetadata(log),
|
|
233
263
|
};
|
|
234
264
|
index.entries[log.id] = updated;
|
|
235
|
-
await upsertSqliteLogIndexEntry(updated);
|
|
236
|
-
scheduleIndexFlush();
|
|
265
|
+
const sqliteUpdated = await upsertSqliteLogIndexEntry(updated);
|
|
266
|
+
if (!sqliteUpdated) scheduleIndexFlush();
|
|
237
267
|
}
|
|
238
268
|
|
|
239
269
|
// Batch writes: collect pending flushes and write once
|
|
@@ -243,6 +273,7 @@ let indexFlushTimeout: ReturnType<typeof setTimeout> | null = null;
|
|
|
243
273
|
async function flushIndexAsync(): Promise<void> {
|
|
244
274
|
indexFlushScheduled = false;
|
|
245
275
|
indexFlushTimeout = null;
|
|
276
|
+
if ((await listSqliteLogCursorEntries({})) !== null) return;
|
|
246
277
|
const index = await loadIndex();
|
|
247
278
|
await saveIndex(index);
|
|
248
279
|
}
|
|
@@ -261,6 +292,7 @@ export async function flushIndex(): Promise<void> {
|
|
|
261
292
|
indexFlushTimeout = null;
|
|
262
293
|
}
|
|
263
294
|
indexFlushScheduled = false;
|
|
295
|
+
if ((await listSqliteLogCursorEntries({})) !== null) return;
|
|
264
296
|
const index = await loadIndex();
|
|
265
297
|
await saveIndex(index);
|
|
266
298
|
}
|
package/src/proxy/logger.ts
CHANGED
|
@@ -61,6 +61,10 @@ function getInspectorLogPath(): string {
|
|
|
61
61
|
return path.join(getDataDir(), "logs", "inspector.log");
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
export function resolveApplicationLogPath(): string {
|
|
65
|
+
return getInspectorLogPath();
|
|
66
|
+
}
|
|
67
|
+
|
|
64
68
|
function readBoundedInteger(
|
|
65
69
|
name: string,
|
|
66
70
|
fallback: number,
|
|
@@ -4,6 +4,11 @@ import { randomUUID } from "node:crypto";
|
|
|
4
4
|
import { open, unlink, type FileHandle } from "node:fs/promises";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { getDataDir } from "./dataDir";
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_RUNTIME_BUDGET_POLICY,
|
|
9
|
+
type RuntimeBudgetPolicy,
|
|
10
|
+
runtimeBudgetPolicySnapshot,
|
|
11
|
+
} from "~/lib/resourceLimits";
|
|
7
12
|
import {
|
|
8
13
|
ensurePrivateDirectorySync,
|
|
9
14
|
PRIVATE_FILE_MODE,
|
|
@@ -34,6 +39,16 @@ export type RawStreamCaptureOptions = {
|
|
|
34
39
|
globalBudget?: RawStreamObservationBudget;
|
|
35
40
|
};
|
|
36
41
|
|
|
42
|
+
export function rawStreamCaptureOptionsFromBudgetPolicy(
|
|
43
|
+
policy: RuntimeBudgetPolicy = DEFAULT_RUNTIME_BUDGET_POLICY,
|
|
44
|
+
): Pick<RawStreamCaptureOptions, "queueMaxMessages" | "queueMaxBytes"> {
|
|
45
|
+
const snapshot = runtimeBudgetPolicySnapshot(policy);
|
|
46
|
+
return {
|
|
47
|
+
queueMaxMessages: snapshot.queues.rawStreamMessages,
|
|
48
|
+
queueMaxBytes: snapshot.queues.rawStreamBytes,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
37
52
|
export type RawStreamSource =
|
|
38
53
|
| {
|
|
39
54
|
type: "memory";
|
|
@@ -108,7 +123,7 @@ const globalObservationBudget = new RawStreamObservationBudget(
|
|
|
108
123
|
),
|
|
109
124
|
);
|
|
110
125
|
|
|
111
|
-
function getRawStreamDir(): string {
|
|
126
|
+
export function getRawStreamDir(): string {
|
|
112
127
|
return join(getDataDir(), "raw-streams");
|
|
113
128
|
}
|
|
114
129
|
|
|
@@ -8,6 +8,7 @@ import { isSqliteLogIndexReady } from "./sqliteLogIndex";
|
|
|
8
8
|
import { recoverCapturedLogStorage } from "./jsonlRecovery";
|
|
9
9
|
import { resolveLogDir } from "./logger";
|
|
10
10
|
import { loadIndex } from "./logIndex";
|
|
11
|
+
import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
|
|
11
12
|
|
|
12
13
|
const DEFAULT_CACHE_MS = 2_000;
|
|
13
14
|
const DEFAULT_MIN_DISK_HEADROOM_BYTES = 256 * 1024 * 1024;
|
|
@@ -86,6 +87,7 @@ function finalizerChecks(): {
|
|
|
86
87
|
queue: ReadinessCheckLevel;
|
|
87
88
|
} {
|
|
88
89
|
const health = getFinalizerRuntimeHealth();
|
|
90
|
+
const sessionQueue = getSessionRuntimeQueueHealth();
|
|
89
91
|
const runtimeArtifactMissing = health.runtime !== "inline" && !health.artifactAvailable;
|
|
90
92
|
const initializedWorkerMissing =
|
|
91
93
|
health.runtime === "worker" &&
|
|
@@ -104,9 +106,9 @@ function finalizerChecks(): {
|
|
|
104
106
|
);
|
|
105
107
|
const warningThreshold = Math.max(1, Math.floor(queueLimit * 0.8));
|
|
106
108
|
const queue =
|
|
107
|
-
health.pendingJobs >= queueLimit
|
|
109
|
+
health.pendingJobs + sessionQueue.queuedTasks >= queueLimit
|
|
108
110
|
? "unavailable"
|
|
109
|
-
: health.pendingJobs >= warningThreshold
|
|
111
|
+
: health.pendingJobs + sessionQueue.queuedTasks >= warningThreshold
|
|
110
112
|
? "degraded"
|
|
111
113
|
: "ok";
|
|
112
114
|
return { workers, queue };
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
|
|
2
|
+
import { getFinalizerRuntimeHealth } from "./logFinalizer";
|
|
3
|
+
import { getRuntimeAdmissionState } from "./runtimeAdmission";
|
|
4
|
+
import { getRuntimeReadiness } from "./runtimeHealth";
|
|
5
|
+
import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
|
|
6
|
+
import { listSqliteLogCursorEntries } from "./sqliteLogIndex";
|
|
7
|
+
import { getStorageLifecycleInventory, getLastStorageLifecycleGcResult } from "./storageLifecycle";
|
|
8
|
+
|
|
9
|
+
export const PERFORMANCE_METRICS_SCHEMA = "agent-inspector.performance-metrics/v1";
|
|
10
|
+
export const OPERATIONAL_METRICS_SCHEMA = "agent-inspector.operational-metrics/v1";
|
|
11
|
+
|
|
12
|
+
export type SseRuntimeMetrics = {
|
|
13
|
+
activeClients: number;
|
|
14
|
+
capacity: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export type SafeOperationalMetrics = {
|
|
18
|
+
schema: typeof PERFORMANCE_METRICS_SCHEMA;
|
|
19
|
+
activeUpstream: number;
|
|
20
|
+
queueDepth: number;
|
|
21
|
+
queueBytes: number;
|
|
22
|
+
incompleteCaptures: number;
|
|
23
|
+
finalizerLagMs: number;
|
|
24
|
+
persistenceLagMs: number;
|
|
25
|
+
activeLeases: number;
|
|
26
|
+
runtime: {
|
|
27
|
+
schema: typeof OPERATIONAL_METRICS_SCHEMA;
|
|
28
|
+
collectedAt: string;
|
|
29
|
+
readiness: Awaited<ReturnType<typeof getRuntimeReadiness>>;
|
|
30
|
+
requests: {
|
|
31
|
+
activeProxyRequests: number;
|
|
32
|
+
proxyCapacity: number;
|
|
33
|
+
acceptingRequests: boolean;
|
|
34
|
+
};
|
|
35
|
+
limits: typeof DEFAULT_RUNTIME_BUDGET_POLICY;
|
|
36
|
+
queues: {
|
|
37
|
+
finalizerPendingJobs: number;
|
|
38
|
+
sessionQueuedTasks: number;
|
|
39
|
+
sessionActiveQueues: number;
|
|
40
|
+
sessionQueueLimit: number;
|
|
41
|
+
};
|
|
42
|
+
workers: ReturnType<typeof getFinalizerRuntimeHealth>;
|
|
43
|
+
storage: {
|
|
44
|
+
totalFiles: number;
|
|
45
|
+
totalBytes: number;
|
|
46
|
+
reclaimableBytes: number;
|
|
47
|
+
lastGc: {
|
|
48
|
+
checkedAt: string;
|
|
49
|
+
filesDeleted: number;
|
|
50
|
+
bytesDeleted: number;
|
|
51
|
+
errors: number;
|
|
52
|
+
} | null;
|
|
53
|
+
};
|
|
54
|
+
sse: SseRuntimeMetrics;
|
|
55
|
+
truncation: {
|
|
56
|
+
captureIncompleteLogs: number;
|
|
57
|
+
droppedChunks: number;
|
|
58
|
+
droppedBytes: number;
|
|
59
|
+
source: "sqlite" | "unavailable";
|
|
60
|
+
};
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
async function getTruncationMetrics(): Promise<SafeOperationalMetrics["runtime"]["truncation"]> {
|
|
65
|
+
const entries = await listSqliteLogCursorEntries({});
|
|
66
|
+
if (entries === null) {
|
|
67
|
+
return {
|
|
68
|
+
captureIncompleteLogs: 0,
|
|
69
|
+
droppedChunks: 0,
|
|
70
|
+
droppedBytes: 0,
|
|
71
|
+
source: "unavailable",
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
captureIncompleteLogs: entries.filter((entry) => entry.summary?.captureIncomplete === true)
|
|
77
|
+
.length,
|
|
78
|
+
droppedChunks: entries.reduce((sum, entry) => sum + (entry.summary?.droppedChunks ?? 0), 0),
|
|
79
|
+
droppedBytes: entries.reduce((sum, entry) => sum + (entry.summary?.droppedBytes ?? 0), 0),
|
|
80
|
+
source: "sqlite",
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export async function getSafeOperationalMetrics(
|
|
85
|
+
sse: SseRuntimeMetrics,
|
|
86
|
+
): Promise<SafeOperationalMetrics> {
|
|
87
|
+
const [readiness, truncation] = await Promise.all([
|
|
88
|
+
getRuntimeReadiness(),
|
|
89
|
+
getTruncationMetrics(),
|
|
90
|
+
]);
|
|
91
|
+
const admission = getRuntimeAdmissionState();
|
|
92
|
+
const workers = getFinalizerRuntimeHealth();
|
|
93
|
+
const sessionQueue = getSessionRuntimeQueueHealth();
|
|
94
|
+
const storageInventory = getStorageLifecycleInventory();
|
|
95
|
+
const lastGc = getLastStorageLifecycleGcResult();
|
|
96
|
+
const reclaimableBytes = storageInventory.artifacts
|
|
97
|
+
.filter(
|
|
98
|
+
(artifact) =>
|
|
99
|
+
artifact.cleanupBehavior === "retention-gc" ||
|
|
100
|
+
artifact.cleanupBehavior === "orphan-reclaim",
|
|
101
|
+
)
|
|
102
|
+
.reduce((sum, artifact) => sum + artifact.bytes, 0);
|
|
103
|
+
const queueDepth = workers.pendingJobs + sessionQueue.queuedTasks;
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
schema: PERFORMANCE_METRICS_SCHEMA,
|
|
107
|
+
activeUpstream: admission.activeRequests,
|
|
108
|
+
queueDepth,
|
|
109
|
+
queueBytes: 0,
|
|
110
|
+
incompleteCaptures: truncation.captureIncompleteLogs,
|
|
111
|
+
finalizerLagMs: 0,
|
|
112
|
+
persistenceLagMs: 0,
|
|
113
|
+
activeLeases: sse.activeClients,
|
|
114
|
+
runtime: {
|
|
115
|
+
schema: OPERATIONAL_METRICS_SCHEMA,
|
|
116
|
+
collectedAt: new Date().toISOString(),
|
|
117
|
+
readiness,
|
|
118
|
+
requests: {
|
|
119
|
+
activeProxyRequests: admission.activeRequests,
|
|
120
|
+
proxyCapacity: admission.capacity,
|
|
121
|
+
acceptingRequests: admission.acceptingRequests,
|
|
122
|
+
},
|
|
123
|
+
limits: DEFAULT_RUNTIME_BUDGET_POLICY,
|
|
124
|
+
queues: {
|
|
125
|
+
finalizerPendingJobs: workers.pendingJobs,
|
|
126
|
+
sessionQueuedTasks: sessionQueue.queuedTasks,
|
|
127
|
+
sessionActiveQueues: sessionQueue.activeQueues,
|
|
128
|
+
sessionQueueLimit: sessionQueue.queueLimit,
|
|
129
|
+
},
|
|
130
|
+
workers,
|
|
131
|
+
storage: {
|
|
132
|
+
totalFiles: storageInventory.totalFiles,
|
|
133
|
+
totalBytes: storageInventory.totalBytes,
|
|
134
|
+
reclaimableBytes,
|
|
135
|
+
lastGc:
|
|
136
|
+
lastGc === null
|
|
137
|
+
? null
|
|
138
|
+
: {
|
|
139
|
+
checkedAt: lastGc.checkedAt,
|
|
140
|
+
filesDeleted: lastGc.filesDeleted,
|
|
141
|
+
bytesDeleted: lastGc.bytesDeleted,
|
|
142
|
+
errors: lastGc.errors.length,
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
sse,
|
|
146
|
+
truncation,
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { logger } from "./logger";
|
|
2
2
|
import { executeFinalizeLogJob, type FinalizeLogJob } from "./logFinalizer";
|
|
3
|
+
import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
|
|
3
4
|
import {
|
|
4
5
|
getLogSessionId,
|
|
5
6
|
markSessionTaskFinished,
|
|
@@ -14,11 +15,25 @@ export type SessionTaskName = "finalize-log";
|
|
|
14
15
|
type SessionTask<T> = () => Promise<T> | T;
|
|
15
16
|
|
|
16
17
|
const queues = new Map<string, Promise<void>>();
|
|
18
|
+
let queuedTaskCount = 0;
|
|
19
|
+
|
|
20
|
+
export class SessionTaskQueueRejectedError extends Error {
|
|
21
|
+
constructor(limit: number) {
|
|
22
|
+
super(`Session task queue is full; limit=${String(limit)}`);
|
|
23
|
+
this.name = "SessionTaskQueueRejectedError";
|
|
24
|
+
}
|
|
25
|
+
}
|
|
17
26
|
|
|
18
27
|
function queueKeyForSession(sessionId: string | null): string {
|
|
19
28
|
return sessionId ?? UNASSIGNED_SESSION_QUEUE;
|
|
20
29
|
}
|
|
21
30
|
|
|
31
|
+
function resolveSessionTaskQueueLimit(): number {
|
|
32
|
+
const configured = Number(process.env["SESSION_TASK_QUEUE_LIMIT"]);
|
|
33
|
+
if (Number.isSafeInteger(configured) && configured > 0) return configured;
|
|
34
|
+
return DEFAULT_RUNTIME_BUDGET_POLICY.queues.finalizerJobs;
|
|
35
|
+
}
|
|
36
|
+
|
|
22
37
|
function errorMessage(err: unknown): string {
|
|
23
38
|
return err instanceof Error ? err.message : String(err);
|
|
24
39
|
}
|
|
@@ -46,9 +61,19 @@ export function enqueueSessionTask<T>(
|
|
|
46
61
|
taskName: SessionTaskName,
|
|
47
62
|
task: SessionTask<T>,
|
|
48
63
|
): Promise<T> {
|
|
64
|
+
const queueLimit = resolveSessionTaskQueueLimit();
|
|
65
|
+
if (queuedTaskCount >= queueLimit) {
|
|
66
|
+
logger.warn(
|
|
67
|
+
`[sessionRuntime] Rejecting ${taskName} for session ${sessionId ?? "unassigned"}: ` +
|
|
68
|
+
`queue full (${String(queuedTaskCount)}/${String(queueLimit)})`,
|
|
69
|
+
);
|
|
70
|
+
return Promise.reject(new SessionTaskQueueRejectedError(queueLimit));
|
|
71
|
+
}
|
|
72
|
+
|
|
49
73
|
const queueKey = queueKeyForSession(sessionId);
|
|
50
74
|
const previous = queues.get(queueKey) ?? Promise.resolve();
|
|
51
75
|
|
|
76
|
+
queuedTaskCount += 1;
|
|
52
77
|
markSessionTaskQueued(sessionId);
|
|
53
78
|
|
|
54
79
|
const current = previous
|
|
@@ -61,6 +86,7 @@ export function enqueueSessionTask<T>(
|
|
|
61
86
|
|
|
62
87
|
queues.set(queueKey, settled);
|
|
63
88
|
void settled.then(() => {
|
|
89
|
+
queuedTaskCount = Math.max(0, queuedTaskCount - 1);
|
|
64
90
|
if (queues.get(queueKey) === settled) {
|
|
65
91
|
queues.delete(queueKey);
|
|
66
92
|
}
|
|
@@ -81,3 +107,15 @@ export async function flushSessionTasks(): Promise<void> {
|
|
|
81
107
|
await Promise.all([...queues.values()]);
|
|
82
108
|
}
|
|
83
109
|
}
|
|
110
|
+
|
|
111
|
+
export function getSessionRuntimeQueueHealth(): {
|
|
112
|
+
activeQueues: number;
|
|
113
|
+
queuedTasks: number;
|
|
114
|
+
queueLimit: number;
|
|
115
|
+
} {
|
|
116
|
+
return {
|
|
117
|
+
activeQueues: queues.size,
|
|
118
|
+
queuedTasks: queuedTaskCount,
|
|
119
|
+
queueLimit: resolveSessionTaskQueueLimit(),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -627,6 +627,57 @@ export async function upsertSqliteLogIndexEntry(entry: LogIndexEntry): Promise<b
|
|
|
627
627
|
return runPrepared(state.db, UPSERT_SQL, entryParams(entry));
|
|
628
628
|
}
|
|
629
629
|
|
|
630
|
+
export type SqliteLogCursorEntryFilters = {
|
|
631
|
+
sessionId?: string;
|
|
632
|
+
model?: string;
|
|
633
|
+
clientPid?: number;
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
function sqliteCursorWhereClause(filters: SqliteLogCursorEntryFilters): {
|
|
637
|
+
sql: string;
|
|
638
|
+
params: unknown[];
|
|
639
|
+
} {
|
|
640
|
+
const clauses: string[] = [];
|
|
641
|
+
const params: unknown[] = [];
|
|
642
|
+
if (filters.sessionId !== undefined) {
|
|
643
|
+
clauses.push("session_id = ?");
|
|
644
|
+
params.push(filters.sessionId);
|
|
645
|
+
}
|
|
646
|
+
if (filters.model !== undefined) {
|
|
647
|
+
clauses.push("model = ?");
|
|
648
|
+
params.push(filters.model);
|
|
649
|
+
}
|
|
650
|
+
if (filters.clientPid !== undefined) {
|
|
651
|
+
clauses.push("client_pid = ?");
|
|
652
|
+
params.push(filters.clientPid);
|
|
653
|
+
}
|
|
654
|
+
return {
|
|
655
|
+
sql: clauses.length === 0 ? "" : ` WHERE ${clauses.join(" AND ")}`,
|
|
656
|
+
params,
|
|
657
|
+
};
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
export async function listSqliteLogCursorEntries(
|
|
661
|
+
filters: SqliteLogCursorEntryFilters,
|
|
662
|
+
): Promise<LogIndexEntry[] | null> {
|
|
663
|
+
const state = await getSqliteState();
|
|
664
|
+
if (state.status !== "ready") return null;
|
|
665
|
+
const where = sqliteCursorWhereClause(filters);
|
|
666
|
+
const rows = allPrepared(
|
|
667
|
+
state.db,
|
|
668
|
+
`SELECT * FROM log_index${where.sql} ORDER BY id ASC`,
|
|
669
|
+
where.params,
|
|
670
|
+
);
|
|
671
|
+
if (rows === null) return null;
|
|
672
|
+
const entries: LogIndexEntry[] = [];
|
|
673
|
+
for (const row of rows) {
|
|
674
|
+
const entry = entryFromRow(row);
|
|
675
|
+
if (entry === null || entry.summary === undefined) return null;
|
|
676
|
+
entries.push(entry);
|
|
677
|
+
}
|
|
678
|
+
return entries;
|
|
679
|
+
}
|
|
680
|
+
|
|
630
681
|
export async function syncSqliteLogIndexEntries(
|
|
631
682
|
entries: readonly LogIndexEntry[],
|
|
632
683
|
): Promise<boolean> {
|