@tonyclaw/agent-inspector 3.1.20 → 3.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/.output/backend/nitro.json +1 -1
  2. package/.output/cli.js +137 -20
  3. package/.output/server/_ssr/index.mjs +2 -2
  4. package/.output/server/_ssr/{router-DpXchCXm.mjs → router-Be6BLuut.mjs} +2371 -1171
  5. package/.output/server/_tanstack-start-manifest_v-ClTaMkEj.mjs +4 -0
  6. package/.output/server/index.mjs +1 -1
  7. package/.output/ui/assets/{CompareDrawer-4i1QHaQf.js → CompareDrawer-CGk2HFI1.js} +1 -1
  8. package/.output/ui/assets/{InspectorPet-Cw6VJLr7.js → InspectorPet-CDM7SdOT.js} +1 -1
  9. package/.output/ui/assets/ProxyViewerContainer-CaWD78tw.js +60 -0
  10. package/.output/ui/assets/{ReplayDialog-DTs575iW.js → ReplayDialog-DqaeV1He.js} +1 -1
  11. package/.output/ui/assets/{RequestAnatomy-Cl6Es3gv.js → RequestAnatomy-BtHl9P0k.js} +1 -1
  12. package/.output/ui/assets/{ResponseView-DNCCLkzG.js → ResponseView-C6t1vpxs.js} +1 -1
  13. package/.output/ui/assets/{StreamingChunkSequence-DKaxRL9U.js → StreamingChunkSequence-R4mgd3GQ.js} +1 -1
  14. package/.output/ui/assets/{_sessionId-C-YjWz50.js → _sessionId-CL3hdLlQ.js} +1 -1
  15. package/.output/ui/assets/{_sessionId-rkR3WgRq.js → _sessionId-HS666S72.js} +1 -1
  16. package/.output/ui/assets/index-B-Qod-aA.css +1 -0
  17. package/.output/ui/assets/{index-DxiecWHw.js → index-Bd9jPL4n.js} +1 -1
  18. package/.output/ui/assets/{index-DYbzZTDM.js → index-C4NHrclw.js} +1 -1
  19. package/.output/ui/assets/{index-D0d-V67x.js → index-UF1PJEcq.js} +1 -1
  20. package/.output/ui/assets/{index-B48RD1-5.js → index-tX38BNV2.js} +2 -2
  21. package/.output/ui/assets/{json-viewer-CjAwZhHA.js → json-viewer-KSgzN_Ov.js} +1 -1
  22. package/.output/ui/assets/{jszip.min-DEGEN1HP.js → jszip.min-BiMDxghN.js} +1 -1
  23. package/.output/ui/index.html +2 -2
  24. package/.output/workers/logFinalizer.worker.js +258 -0
  25. package/.output/workers/sessionWorkerEntry.js +258 -0
  26. package/package.json +13 -6
  27. package/src/backend/routes/api/logs.$id.replay.ts +6 -1
  28. package/src/backend/routes/api/logs.stream.ts +7 -0
  29. package/src/backend/routes/api/providers.$providerId.model-metadata.ts +15 -4
  30. package/src/backend/routes/api/storage.ts +56 -0
  31. package/src/backend/routes/metrics.ts +14 -0
  32. package/src/cli/doctor.ts +135 -2
  33. package/src/cli.ts +7 -4
  34. package/src/components/ProxyViewerContainer.tsx +41 -24
  35. package/src/components/providers/SettingsDialog.tsx +261 -70
  36. package/src/components/proxy-viewer/LogEntry.tsx +19 -5
  37. package/src/components/proxy-viewer/bodyHydration.ts +67 -0
  38. package/src/components/proxy-viewer/proxyViewerContainerLogic.ts +78 -0
  39. package/src/knowledge/openclawClient.ts +11 -2
  40. package/src/knowledge/openclawGatewayClient.ts +6 -1
  41. package/src/lib/resourceLimits.ts +191 -0
  42. package/src/lib/safeFetch.ts +428 -20
  43. package/src/proxy/ecosystemTasks.ts +20 -0
  44. package/src/proxy/handler.ts +17 -4
  45. package/src/proxy/identityProxy.ts +36 -16
  46. package/src/proxy/logIndex.ts +38 -6
  47. package/src/proxy/logger.ts +4 -0
  48. package/src/proxy/rawStreamCapture.ts +16 -1
  49. package/src/proxy/runtimeHealth.ts +4 -2
  50. package/src/proxy/runtimeMetrics.ts +149 -0
  51. package/src/proxy/sessionRuntime.ts +38 -0
  52. package/src/proxy/sqliteLogIndex.ts +51 -0
  53. package/src/proxy/storageLifecycle.ts +432 -0
  54. package/src/proxy/store.ts +62 -12
  55. package/.output/server/_tanstack-start-manifest_v-DnbdNeun.mjs +0 -4
  56. package/.output/ui/assets/ProxyViewerContainer-BST0B7lY.js +0 -60
  57. package/.output/ui/assets/index-IBEDJoV_.css +0 -1
@@ -14,6 +14,7 @@ import {
14
14
  signalProcessTree,
15
15
  type CommandSpec,
16
16
  } from "./platformCommands";
17
+ import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
17
18
 
18
19
  const MAX_TASKS = 50;
19
20
  const OUTPUT_LIMIT = 40_000;
@@ -34,6 +35,12 @@ const activeTasks = new Map<string, ActiveTask>();
34
35
  let acceptingTasks = true;
35
36
  let shutdownForceTimer: ReturnType<typeof setTimeout> | null = null;
36
37
 
38
+ function activeTaskLimit(): number {
39
+ const configured = Number(process.env["ECOSYSTEM_TASK_ACTIVE_LIMIT"]);
40
+ if (Number.isSafeInteger(configured) && configured > 0) return configured;
41
+ return DEFAULT_RUNTIME_BUDGET_POLICY.queues.backgroundWorkers;
42
+ }
43
+
37
44
  function nowIso(): string {
38
45
  return new Date().toISOString();
39
46
  }
@@ -258,6 +265,7 @@ export function startEcosystemTask(
258
265
  action: EcosystemTaskAction,
259
266
  ): EcosystemTask | null {
260
267
  if (!acceptingTasks) return null;
268
+ if (activeTasks.size >= activeTaskLimit()) return null;
261
269
  if (action === "recipe") return null;
262
270
  const definition =
263
271
  action === "runner-presets"
@@ -291,6 +299,7 @@ export function startEcosystemTask(
291
299
 
292
300
  export function startEcosystemRecipeTask(recipeId: string): EcosystemTask | null {
293
301
  if (!acceptingTasks) return null;
302
+ if (activeTasks.size >= activeTaskLimit()) return null;
294
303
  const recipe = findEcosystemRecipe(recipeId);
295
304
  if (recipe === null) return null;
296
305
  if (!recipe.runnable) return null;
@@ -367,6 +376,7 @@ export function _startEcosystemTaskForTests(
367
376
  timeoutMs = RECIPE_TIMEOUT_MS,
368
377
  ): EcosystemTask | null {
369
378
  if (!acceptingTasks) return null;
379
+ if (activeTasks.size >= activeTaskLimit()) return null;
370
380
  const task: EcosystemTask = {
371
381
  id: randomUUID(),
372
382
  packageId: "test-package",
@@ -393,3 +403,13 @@ export function _resetEcosystemTasksForTests(): void {
393
403
  activeTasks.clear();
394
404
  acceptingTasks = true;
395
405
  }
406
+
407
+ export function _getEcosystemTaskAdmissionHealthForTests(): {
408
+ activeTasks: number;
409
+ activeTaskLimit: number;
410
+ } {
411
+ return {
412
+ activeTasks: activeTasks.size,
413
+ activeTaskLimit: activeTaskLimit(),
414
+ };
415
+ }
@@ -40,7 +40,7 @@ import {
40
40
  getRuntimeAdmissionState,
41
41
  type RuntimeRequestLease,
42
42
  } from "./runtimeAdmission";
43
- import { resourceLimitResponse } from "../lib/resourceLimits";
43
+ import { DEFAULT_RUNTIME_BUDGET_POLICY, resourceLimitResponse } from "../lib/resourceLimits";
44
44
  import {
45
45
  buildUpstreamUrl,
46
46
  describeApiRoute,
@@ -592,10 +592,19 @@ async function handleAdmittedProxy(req: Request): Promise<Response> {
592
592
  body: bodyToForward,
593
593
  signal: req.signal,
594
594
  },
595
- { allowLoopback: ALLOW_LOOPBACK },
595
+ {
596
+ allowLoopback: ALLOW_LOOPBACK,
597
+ deadlineMs: DEFAULT_RUNTIME_BUDGET_POLICY.upstream.fetchDeadlineMs,
598
+ responseBytes: DEFAULT_RUNTIME_BUDGET_POLICY.upstream.fetchResponseBytes,
599
+ },
596
600
  );
597
601
  if (!fetchResult.ok) {
598
- const status = fetchResult.kind === "network-policy" ? STATUS_FORBIDDEN : STATUS_BAD_GATEWAY;
602
+ const status =
603
+ fetchResult.kind === "network-policy"
604
+ ? STATUS_FORBIDDEN
605
+ : fetchResult.kind === "resource-limit"
606
+ ? STATUS_PAYLOAD_TOO_LARGE
607
+ : STATUS_BAD_GATEWAY;
599
608
  logger.warn(`[handler] Safe fetch blocked ${upstreamUrl}: ${fetchResult.reason}`);
600
609
  log.elapsedMs = Date.now() - startTime;
601
610
  log.responseStatus = status;
@@ -605,7 +614,11 @@ async function handleAdmittedProxy(req: Request): Promise<Response> {
605
614
  return new Response(
606
615
  fetchResult.kind === "network-policy"
607
616
  ? "Forbidden: upstream URL is not permitted"
608
- : "Proxy error: upstream redirects are not permitted",
617
+ : fetchResult.kind === "resource-limit"
618
+ ? "Proxy error: upstream response exceeded configured limits"
619
+ : fetchResult.kind === "timeout"
620
+ ? "Proxy error: upstream request timed out"
621
+ : "Proxy error: upstream redirects are not permitted",
609
622
  { status },
610
623
  );
611
624
  }
@@ -585,9 +585,10 @@ function normalizedHostname(value: string): string | null {
585
585
  }
586
586
 
587
587
  function isLoopbackHostname(hostname: string): boolean {
588
- if (hostname === "localhost" || hostname === "::1") return true;
589
- if (isIP(hostname) !== 4) return false;
590
- return hostname.split(".", 1)[0] === "127";
588
+ const normalized = hostname.toLowerCase().replace(/^\[|]$/g, "");
589
+ if (normalized === "localhost" || normalized === "::1") return true;
590
+ if (isIP(normalized) !== 4) return false;
591
+ return normalized.split(".", 1)[0] === "127";
591
592
  }
592
593
 
593
594
  export type UiOriginAllowlist = {
@@ -669,15 +670,31 @@ function normalizedRequestOrigin(origin: string): string | null {
669
670
  }
670
671
  }
671
672
 
673
+ function parsedRequestOrigin(origin: string): URL | null {
674
+ const normalized = normalizedRequestOrigin(origin);
675
+ if (normalized === null) return null;
676
+ return new URL(normalized);
677
+ }
678
+
679
+ function isSameOrLoopbackAliasOrigin(left: URL, right: URL): boolean {
680
+ if (left.origin === right.origin) return true;
681
+ return (
682
+ left.protocol === right.protocol &&
683
+ left.port === right.port &&
684
+ isLoopbackHostname(left.hostname) &&
685
+ isLoopbackHostname(right.hostname)
686
+ );
687
+ }
688
+
672
689
  function isSameRequestOrigin(
673
690
  origin: string,
674
691
  host: string | undefined,
675
692
  publicOrigin?: string,
676
693
  ): boolean {
677
- const parsedOrigin = normalizedRequestOrigin(origin);
694
+ const parsedOrigin = parsedRequestOrigin(origin);
678
695
  if (parsedOrigin === null) return false;
679
- const expected = normalizedRequestOrigin(publicOrigin ?? `http://${host ?? "localhost"}`);
680
- return expected !== null && parsedOrigin === expected;
696
+ const expected = parsedRequestOrigin(publicOrigin ?? `http://${host ?? "localhost"}`);
697
+ return expected !== null && isSameOrLoopbackAliasOrigin(parsedOrigin, expected);
681
698
  }
682
699
 
683
700
  function matchedUiOrigin(
@@ -797,24 +814,27 @@ export function isBrowserMutationAllowed(input: {
797
814
  const fetchSite = input.secFetchSite?.toLowerCase();
798
815
  const browserRequest = input.origin !== undefined || fetchSite !== undefined;
799
816
  if (!browserRequest) return true;
817
+ const sameOrigin =
818
+ input.origin !== undefined
819
+ ? isSameRequestOrigin(input.origin, input.host, input.publicOrigin)
820
+ : false;
800
821
  if (isModelProxyPath(pathname)) {
801
- if (fetchSite !== undefined && fetchSite !== "same-origin") return false;
802
- if (
803
- input.origin !== undefined &&
804
- !isSameRequestOrigin(input.origin, input.host, input.publicOrigin)
805
- ) {
822
+ if (fetchSite !== undefined && fetchSite !== "same-origin" && !sameOrigin) return false;
823
+ if (input.origin !== undefined && !sameOrigin) {
806
824
  return false;
807
825
  }
808
826
  return input.csrfToken === "1";
809
827
  }
810
828
  const allowedOrigin = matchedUiOrigin(input.origin, input.allowedUiOrigins);
811
- if (fetchSite !== undefined && fetchSite !== "same-origin" && allowedOrigin === null)
829
+ if (
830
+ fetchSite !== undefined &&
831
+ fetchSite !== "same-origin" &&
832
+ !sameOrigin &&
833
+ allowedOrigin === null
834
+ )
812
835
  return false;
813
836
  if (input.origin !== undefined) {
814
- if (
815
- !isSameRequestOrigin(input.origin, input.host, input.publicOrigin) &&
816
- allowedOrigin === null
817
- ) {
837
+ if (!sameOrigin && allowedOrigin === null) {
818
838
  return false;
819
839
  }
820
840
  }
@@ -17,6 +17,7 @@ import { extractToolTraceEvents } from "../lib/toolTrace";
17
17
  import { extractStopReason } from "../lib/stopReason";
18
18
  import {
19
19
  getSqliteLogIndexMaxId,
20
+ listSqliteLogCursorEntries,
20
21
  replaceSqliteLogIndexEntries,
21
22
  syncSqliteLogIndexEntries,
22
23
  upsertSqliteLogIndexEntry,
@@ -159,9 +160,36 @@ function createEmptyIndex(): LogIndex {
159
160
  };
160
161
  }
161
162
 
163
+ function createIndexFromEntries(entries: readonly LogIndexEntry[]): LogIndex {
164
+ const index = createEmptyIndex();
165
+ for (const entry of entries) {
166
+ index.entries[entry.id] = entry;
167
+ index.maxId = Math.max(index.maxId, entry.id);
168
+ }
169
+ return index;
170
+ }
171
+
172
+ function sqliteProjectionReferencesExistingLedgers(entries: readonly LogIndexEntry[]): boolean {
173
+ const logDir = resolveLogDir();
174
+ for (const entry of entries) {
175
+ if (!entry.file.endsWith(".jsonl")) return false;
176
+ if (!existsSync(join(logDir, entry.file))) return false;
177
+ }
178
+ return true;
179
+ }
180
+
162
181
  export async function loadIndex(): Promise<LogIndex> {
163
182
  if (cachedIndex !== null) return cachedIndex;
164
- // logs.idx and inspector.sqlite are projections only. A cold process always rebuilds from
183
+ const sqliteEntries = await listSqliteLogCursorEntries({});
184
+ if (sqliteEntries !== null) {
185
+ if (sqliteProjectionReferencesExistingLedgers(sqliteEntries)) {
186
+ cachedIndex = createIndexFromEntries(sqliteEntries);
187
+ allocatedMaxId = Math.max(allocatedMaxId, cachedIndex.maxId);
188
+ return cachedIndex;
189
+ }
190
+ logger.warn("[logIndex] SQLite projection references missing ledgers; rebuilding from JSONL");
191
+ }
192
+ // logs.idx remains a legacy fallback projection only. If SQLite is unavailable, rebuild from
165
193
  // authoritative JSONL rather than allowing a stale, conflicting, or higher derived row to win.
166
194
  return await rebuildIndex();
167
195
  }
@@ -212,9 +240,11 @@ export async function addToIndex(
212
240
  if (id > index.maxId) {
213
241
  index.maxId = id;
214
242
  }
215
- void upsertSqliteLogIndexEntry(entry);
216
- // Defer disk writes to reduce I/O - flush after a batch of updates
217
- scheduleIndexFlush();
243
+ const sqliteUpdated = await upsertSqliteLogIndexEntry(entry);
244
+ if (!sqliteUpdated) {
245
+ // Legacy fallback only: defer disk writes to reduce I/O when SQLite is unavailable.
246
+ scheduleIndexFlush();
247
+ }
218
248
  }
219
249
 
220
250
  /**
@@ -232,8 +262,8 @@ export async function updateLogIndexEntryMetadata(log: CapturedLog): Promise<voi
232
262
  ...createLogIndexEntryMetadata(log),
233
263
  };
234
264
  index.entries[log.id] = updated;
235
- await upsertSqliteLogIndexEntry(updated);
236
- scheduleIndexFlush();
265
+ const sqliteUpdated = await upsertSqliteLogIndexEntry(updated);
266
+ if (!sqliteUpdated) scheduleIndexFlush();
237
267
  }
238
268
 
239
269
  // Batch writes: collect pending flushes and write once
@@ -243,6 +273,7 @@ let indexFlushTimeout: ReturnType<typeof setTimeout> | null = null;
243
273
  async function flushIndexAsync(): Promise<void> {
244
274
  indexFlushScheduled = false;
245
275
  indexFlushTimeout = null;
276
+ if ((await listSqliteLogCursorEntries({})) !== null) return;
246
277
  const index = await loadIndex();
247
278
  await saveIndex(index);
248
279
  }
@@ -261,6 +292,7 @@ export async function flushIndex(): Promise<void> {
261
292
  indexFlushTimeout = null;
262
293
  }
263
294
  indexFlushScheduled = false;
295
+ if ((await listSqliteLogCursorEntries({})) !== null) return;
264
296
  const index = await loadIndex();
265
297
  await saveIndex(index);
266
298
  }
@@ -61,6 +61,10 @@ function getInspectorLogPath(): string {
61
61
  return path.join(getDataDir(), "logs", "inspector.log");
62
62
  }
63
63
 
64
+ export function resolveApplicationLogPath(): string {
65
+ return getInspectorLogPath();
66
+ }
67
+
64
68
  function readBoundedInteger(
65
69
  name: string,
66
70
  fallback: number,
@@ -4,6 +4,11 @@ import { randomUUID } from "node:crypto";
4
4
  import { open, unlink, type FileHandle } from "node:fs/promises";
5
5
  import { join } from "node:path";
6
6
  import { getDataDir } from "./dataDir";
7
+ import {
8
+ DEFAULT_RUNTIME_BUDGET_POLICY,
9
+ type RuntimeBudgetPolicy,
10
+ runtimeBudgetPolicySnapshot,
11
+ } from "~/lib/resourceLimits";
7
12
  import {
8
13
  ensurePrivateDirectorySync,
9
14
  PRIVATE_FILE_MODE,
@@ -34,6 +39,16 @@ export type RawStreamCaptureOptions = {
34
39
  globalBudget?: RawStreamObservationBudget;
35
40
  };
36
41
 
42
+ export function rawStreamCaptureOptionsFromBudgetPolicy(
43
+ policy: RuntimeBudgetPolicy = DEFAULT_RUNTIME_BUDGET_POLICY,
44
+ ): Pick<RawStreamCaptureOptions, "queueMaxMessages" | "queueMaxBytes"> {
45
+ const snapshot = runtimeBudgetPolicySnapshot(policy);
46
+ return {
47
+ queueMaxMessages: snapshot.queues.rawStreamMessages,
48
+ queueMaxBytes: snapshot.queues.rawStreamBytes,
49
+ };
50
+ }
51
+
37
52
  export type RawStreamSource =
38
53
  | {
39
54
  type: "memory";
@@ -108,7 +123,7 @@ const globalObservationBudget = new RawStreamObservationBudget(
108
123
  ),
109
124
  );
110
125
 
111
- function getRawStreamDir(): string {
126
+ export function getRawStreamDir(): string {
112
127
  return join(getDataDir(), "raw-streams");
113
128
  }
114
129
 
@@ -8,6 +8,7 @@ import { isSqliteLogIndexReady } from "./sqliteLogIndex";
8
8
  import { recoverCapturedLogStorage } from "./jsonlRecovery";
9
9
  import { resolveLogDir } from "./logger";
10
10
  import { loadIndex } from "./logIndex";
11
+ import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
11
12
 
12
13
  const DEFAULT_CACHE_MS = 2_000;
13
14
  const DEFAULT_MIN_DISK_HEADROOM_BYTES = 256 * 1024 * 1024;
@@ -86,6 +87,7 @@ function finalizerChecks(): {
86
87
  queue: ReadinessCheckLevel;
87
88
  } {
88
89
  const health = getFinalizerRuntimeHealth();
90
+ const sessionQueue = getSessionRuntimeQueueHealth();
89
91
  const runtimeArtifactMissing = health.runtime !== "inline" && !health.artifactAvailable;
90
92
  const initializedWorkerMissing =
91
93
  health.runtime === "worker" &&
@@ -104,9 +106,9 @@ function finalizerChecks(): {
104
106
  );
105
107
  const warningThreshold = Math.max(1, Math.floor(queueLimit * 0.8));
106
108
  const queue =
107
- health.pendingJobs >= queueLimit
109
+ health.pendingJobs + sessionQueue.queuedTasks >= queueLimit
108
110
  ? "unavailable"
109
- : health.pendingJobs >= warningThreshold
111
+ : health.pendingJobs + sessionQueue.queuedTasks >= warningThreshold
110
112
  ? "degraded"
111
113
  : "ok";
112
114
  return { workers, queue };
@@ -0,0 +1,149 @@
1
+ import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
2
+ import { getFinalizerRuntimeHealth } from "./logFinalizer";
3
+ import { getRuntimeAdmissionState } from "./runtimeAdmission";
4
+ import { getRuntimeReadiness } from "./runtimeHealth";
5
+ import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
6
+ import { listSqliteLogCursorEntries } from "./sqliteLogIndex";
7
+ import { getStorageLifecycleInventory, getLastStorageLifecycleGcResult } from "./storageLifecycle";
8
+
9
+ export const PERFORMANCE_METRICS_SCHEMA = "agent-inspector.performance-metrics/v1";
10
+ export const OPERATIONAL_METRICS_SCHEMA = "agent-inspector.operational-metrics/v1";
11
+
12
+ export type SseRuntimeMetrics = {
13
+ activeClients: number;
14
+ capacity: number;
15
+ };
16
+
17
+ export type SafeOperationalMetrics = {
18
+ schema: typeof PERFORMANCE_METRICS_SCHEMA;
19
+ activeUpstream: number;
20
+ queueDepth: number;
21
+ queueBytes: number;
22
+ incompleteCaptures: number;
23
+ finalizerLagMs: number;
24
+ persistenceLagMs: number;
25
+ activeLeases: number;
26
+ runtime: {
27
+ schema: typeof OPERATIONAL_METRICS_SCHEMA;
28
+ collectedAt: string;
29
+ readiness: Awaited<ReturnType<typeof getRuntimeReadiness>>;
30
+ requests: {
31
+ activeProxyRequests: number;
32
+ proxyCapacity: number;
33
+ acceptingRequests: boolean;
34
+ };
35
+ limits: typeof DEFAULT_RUNTIME_BUDGET_POLICY;
36
+ queues: {
37
+ finalizerPendingJobs: number;
38
+ sessionQueuedTasks: number;
39
+ sessionActiveQueues: number;
40
+ sessionQueueLimit: number;
41
+ };
42
+ workers: ReturnType<typeof getFinalizerRuntimeHealth>;
43
+ storage: {
44
+ totalFiles: number;
45
+ totalBytes: number;
46
+ reclaimableBytes: number;
47
+ lastGc: {
48
+ checkedAt: string;
49
+ filesDeleted: number;
50
+ bytesDeleted: number;
51
+ errors: number;
52
+ } | null;
53
+ };
54
+ sse: SseRuntimeMetrics;
55
+ truncation: {
56
+ captureIncompleteLogs: number;
57
+ droppedChunks: number;
58
+ droppedBytes: number;
59
+ source: "sqlite" | "unavailable";
60
+ };
61
+ };
62
+ };
63
+
64
+ async function getTruncationMetrics(): Promise<SafeOperationalMetrics["runtime"]["truncation"]> {
65
+ const entries = await listSqliteLogCursorEntries({});
66
+ if (entries === null) {
67
+ return {
68
+ captureIncompleteLogs: 0,
69
+ droppedChunks: 0,
70
+ droppedBytes: 0,
71
+ source: "unavailable",
72
+ };
73
+ }
74
+
75
+ return {
76
+ captureIncompleteLogs: entries.filter((entry) => entry.summary?.captureIncomplete === true)
77
+ .length,
78
+ droppedChunks: entries.reduce((sum, entry) => sum + (entry.summary?.droppedChunks ?? 0), 0),
79
+ droppedBytes: entries.reduce((sum, entry) => sum + (entry.summary?.droppedBytes ?? 0), 0),
80
+ source: "sqlite",
81
+ };
82
+ }
83
+
84
+ export async function getSafeOperationalMetrics(
85
+ sse: SseRuntimeMetrics,
86
+ ): Promise<SafeOperationalMetrics> {
87
+ const [readiness, truncation] = await Promise.all([
88
+ getRuntimeReadiness(),
89
+ getTruncationMetrics(),
90
+ ]);
91
+ const admission = getRuntimeAdmissionState();
92
+ const workers = getFinalizerRuntimeHealth();
93
+ const sessionQueue = getSessionRuntimeQueueHealth();
94
+ const storageInventory = getStorageLifecycleInventory();
95
+ const lastGc = getLastStorageLifecycleGcResult();
96
+ const reclaimableBytes = storageInventory.artifacts
97
+ .filter(
98
+ (artifact) =>
99
+ artifact.cleanupBehavior === "retention-gc" ||
100
+ artifact.cleanupBehavior === "orphan-reclaim",
101
+ )
102
+ .reduce((sum, artifact) => sum + artifact.bytes, 0);
103
+ const queueDepth = workers.pendingJobs + sessionQueue.queuedTasks;
104
+
105
+ return {
106
+ schema: PERFORMANCE_METRICS_SCHEMA,
107
+ activeUpstream: admission.activeRequests,
108
+ queueDepth,
109
+ queueBytes: 0,
110
+ incompleteCaptures: truncation.captureIncompleteLogs,
111
+ finalizerLagMs: 0,
112
+ persistenceLagMs: 0,
113
+ activeLeases: sse.activeClients,
114
+ runtime: {
115
+ schema: OPERATIONAL_METRICS_SCHEMA,
116
+ collectedAt: new Date().toISOString(),
117
+ readiness,
118
+ requests: {
119
+ activeProxyRequests: admission.activeRequests,
120
+ proxyCapacity: admission.capacity,
121
+ acceptingRequests: admission.acceptingRequests,
122
+ },
123
+ limits: DEFAULT_RUNTIME_BUDGET_POLICY,
124
+ queues: {
125
+ finalizerPendingJobs: workers.pendingJobs,
126
+ sessionQueuedTasks: sessionQueue.queuedTasks,
127
+ sessionActiveQueues: sessionQueue.activeQueues,
128
+ sessionQueueLimit: sessionQueue.queueLimit,
129
+ },
130
+ workers,
131
+ storage: {
132
+ totalFiles: storageInventory.totalFiles,
133
+ totalBytes: storageInventory.totalBytes,
134
+ reclaimableBytes,
135
+ lastGc:
136
+ lastGc === null
137
+ ? null
138
+ : {
139
+ checkedAt: lastGc.checkedAt,
140
+ filesDeleted: lastGc.filesDeleted,
141
+ bytesDeleted: lastGc.bytesDeleted,
142
+ errors: lastGc.errors.length,
143
+ },
144
+ },
145
+ sse,
146
+ truncation,
147
+ },
148
+ };
149
+ }
@@ -1,5 +1,6 @@
1
1
  import { logger } from "./logger";
2
2
  import { executeFinalizeLogJob, type FinalizeLogJob } from "./logFinalizer";
3
+ import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
3
4
  import {
4
5
  getLogSessionId,
5
6
  markSessionTaskFinished,
@@ -14,11 +15,25 @@ export type SessionTaskName = "finalize-log";
14
15
  type SessionTask<T> = () => Promise<T> | T;
15
16
 
16
17
  const queues = new Map<string, Promise<void>>();
18
+ let queuedTaskCount = 0;
19
+
20
+ export class SessionTaskQueueRejectedError extends Error {
21
+ constructor(limit: number) {
22
+ super(`Session task queue is full; limit=${String(limit)}`);
23
+ this.name = "SessionTaskQueueRejectedError";
24
+ }
25
+ }
17
26
 
18
27
  function queueKeyForSession(sessionId: string | null): string {
19
28
  return sessionId ?? UNASSIGNED_SESSION_QUEUE;
20
29
  }
21
30
 
31
+ function resolveSessionTaskQueueLimit(): number {
32
+ const configured = Number(process.env["SESSION_TASK_QUEUE_LIMIT"]);
33
+ if (Number.isSafeInteger(configured) && configured > 0) return configured;
34
+ return DEFAULT_RUNTIME_BUDGET_POLICY.queues.finalizerJobs;
35
+ }
36
+
22
37
  function errorMessage(err: unknown): string {
23
38
  return err instanceof Error ? err.message : String(err);
24
39
  }
@@ -46,9 +61,19 @@ export function enqueueSessionTask<T>(
46
61
  taskName: SessionTaskName,
47
62
  task: SessionTask<T>,
48
63
  ): Promise<T> {
64
+ const queueLimit = resolveSessionTaskQueueLimit();
65
+ if (queuedTaskCount >= queueLimit) {
66
+ logger.warn(
67
+ `[sessionRuntime] Rejecting ${taskName} for session ${sessionId ?? "unassigned"}: ` +
68
+ `queue full (${String(queuedTaskCount)}/${String(queueLimit)})`,
69
+ );
70
+ return Promise.reject(new SessionTaskQueueRejectedError(queueLimit));
71
+ }
72
+
49
73
  const queueKey = queueKeyForSession(sessionId);
50
74
  const previous = queues.get(queueKey) ?? Promise.resolve();
51
75
 
76
+ queuedTaskCount += 1;
52
77
  markSessionTaskQueued(sessionId);
53
78
 
54
79
  const current = previous
@@ -61,6 +86,7 @@ export function enqueueSessionTask<T>(
61
86
 
62
87
  queues.set(queueKey, settled);
63
88
  void settled.then(() => {
89
+ queuedTaskCount = Math.max(0, queuedTaskCount - 1);
64
90
  if (queues.get(queueKey) === settled) {
65
91
  queues.delete(queueKey);
66
92
  }
@@ -81,3 +107,15 @@ export async function flushSessionTasks(): Promise<void> {
81
107
  await Promise.all([...queues.values()]);
82
108
  }
83
109
  }
110
+
111
+ export function getSessionRuntimeQueueHealth(): {
112
+ activeQueues: number;
113
+ queuedTasks: number;
114
+ queueLimit: number;
115
+ } {
116
+ return {
117
+ activeQueues: queues.size,
118
+ queuedTasks: queuedTaskCount,
119
+ queueLimit: resolveSessionTaskQueueLimit(),
120
+ };
121
+ }
@@ -627,6 +627,57 @@ export async function upsertSqliteLogIndexEntry(entry: LogIndexEntry): Promise<b
627
627
  return runPrepared(state.db, UPSERT_SQL, entryParams(entry));
628
628
  }
629
629
 
630
+ export type SqliteLogCursorEntryFilters = {
631
+ sessionId?: string;
632
+ model?: string;
633
+ clientPid?: number;
634
+ };
635
+
636
+ function sqliteCursorWhereClause(filters: SqliteLogCursorEntryFilters): {
637
+ sql: string;
638
+ params: unknown[];
639
+ } {
640
+ const clauses: string[] = [];
641
+ const params: unknown[] = [];
642
+ if (filters.sessionId !== undefined) {
643
+ clauses.push("session_id = ?");
644
+ params.push(filters.sessionId);
645
+ }
646
+ if (filters.model !== undefined) {
647
+ clauses.push("model = ?");
648
+ params.push(filters.model);
649
+ }
650
+ if (filters.clientPid !== undefined) {
651
+ clauses.push("client_pid = ?");
652
+ params.push(filters.clientPid);
653
+ }
654
+ return {
655
+ sql: clauses.length === 0 ? "" : ` WHERE ${clauses.join(" AND ")}`,
656
+ params,
657
+ };
658
+ }
659
+
660
+ export async function listSqliteLogCursorEntries(
661
+ filters: SqliteLogCursorEntryFilters,
662
+ ): Promise<LogIndexEntry[] | null> {
663
+ const state = await getSqliteState();
664
+ if (state.status !== "ready") return null;
665
+ const where = sqliteCursorWhereClause(filters);
666
+ const rows = allPrepared(
667
+ state.db,
668
+ `SELECT * FROM log_index${where.sql} ORDER BY id ASC`,
669
+ where.params,
670
+ );
671
+ if (rows === null) return null;
672
+ const entries: LogIndexEntry[] = [];
673
+ for (const row of rows) {
674
+ const entry = entryFromRow(row);
675
+ if (entry === null || entry.summary === undefined) return null;
676
+ entries.push(entry);
677
+ }
678
+ return entries;
679
+ }
680
+
630
681
  export async function syncSqliteLogIndexEntries(
631
682
  entries: readonly LogIndexEntry[],
632
683
  ): Promise<boolean> {