agent-relay-server 0.120.6 → 0.121.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/docs/openapi.json +1 -1
  2. package/package.json +3 -3
  3. package/public/assets/{MessageBubble-So86mU9-.js → MessageBubble-BaPGFJxq.js} +2 -2
  4. package/public/assets/{MessageBubble-So86mU9-.js.map → MessageBubble-BaPGFJxq.js.map} +1 -1
  5. package/public/assets/{PlanGraphPanel-DBZYAsww.js → PlanGraphPanel-BXCVfmln.js} +2 -2
  6. package/public/assets/{PlanGraphPanel-DBZYAsww.js.map → PlanGraphPanel-BXCVfmln.js.map} +1 -1
  7. package/public/assets/{activity-CZNwmXg1.js → activity-D_shwAZt.js} +2 -2
  8. package/public/assets/{activity-CZNwmXg1.js.map → activity-D_shwAZt.js.map} +1 -1
  9. package/public/assets/{agent-profiles-BhadoGwi.js → agent-profiles-BrfVz5IV.js} +2 -2
  10. package/public/assets/{agent-profiles-BhadoGwi.js.map → agent-profiles-BrfVz5IV.js.map} +1 -1
  11. package/public/assets/{agents-DWn1xx9x.js → agents-B5w-Z9Ic.js} +2 -2
  12. package/public/assets/{agents-DWn1xx9x.js.map → agents-B5w-Z9Ic.js.map} +1 -1
  13. package/public/assets/{analytics-DifGnk7q.js → analytics-Bm3D12UN.js} +2 -2
  14. package/public/assets/{analytics-DifGnk7q.js.map → analytics-Bm3D12UN.js.map} +1 -1
  15. package/public/assets/{automation-BExrtKkb.js → automation-DOTL8BFs.js} +2 -2
  16. package/public/assets/{automation-BExrtKkb.js.map → automation-DOTL8BFs.js.map} +1 -1
  17. package/public/assets/{branch-state-badge-CyC5Qrhc.js → branch-state-badge-BlqeJZ5-.js} +2 -2
  18. package/public/assets/{branch-state-badge-CyC5Qrhc.js.map → branch-state-badge-BlqeJZ5-.js.map} +1 -1
  19. package/public/assets/{channels-BLkdt2XR.js → channels-BoiKfK6Q.js} +2 -2
  20. package/public/assets/{channels-BLkdt2XR.js.map → channels-BoiKfK6Q.js.map} +1 -1
  21. package/public/assets/{chat-C7OhKtr6.js → chat-PeU3iOaa.js} +2 -2
  22. package/public/assets/{chat-C7OhKtr6.js.map → chat-PeU3iOaa.js.map} +1 -1
  23. package/public/assets/{connectors-C0n12Je8.js → connectors-BvV8Hee2.js} +2 -2
  24. package/public/assets/{connectors-C0n12Je8.js.map → connectors-BvV8Hee2.js.map} +1 -1
  25. package/public/assets/{formatted-body-BCQT2qSb.js → formatted-body-C1FgzlT1.js} +3 -3
  26. package/public/assets/{formatted-body-BCQT2qSb.js.map → formatted-body-C1FgzlT1.js.map} +1 -1
  27. package/public/assets/{formatted-body-impl-Bk7wnESl.js → formatted-body-impl-BTpU1xH4.js} +2 -2
  28. package/public/assets/{formatted-body-impl-Bk7wnESl.js.map → formatted-body-impl-BTpU1xH4.js.map} +1 -1
  29. package/public/assets/{index-CRraCeo_.js → index-DZEahmyc.js} +6 -6
  30. package/public/assets/{index-CRraCeo_.js.map → index-DZEahmyc.js.map} +1 -1
  31. package/public/assets/{insights-DEgES6jz.js → insights-BqmgSpQm.js} +2 -2
  32. package/public/assets/{insights-DEgES6jz.js.map → insights-BqmgSpQm.js.map} +1 -1
  33. package/public/assets/{integrations-v14ehskT.js → integrations-CuT92zQi.js} +2 -2
  34. package/public/assets/{integrations-v14ehskT.js.map → integrations-CuT92zQi.js.map} +1 -1
  35. package/public/assets/{maintenance-BgVB6_gO.js → maintenance-lM7FeXVC.js} +2 -2
  36. package/public/assets/{maintenance-BgVB6_gO.js.map → maintenance-lM7FeXVC.js.map} +1 -1
  37. package/public/assets/{managed-agents-8_NKSkhI.js → managed-agents-DaPNv3yY.js} +2 -2
  38. package/public/assets/{managed-agents-8_NKSkhI.js.map → managed-agents-DaPNv3yY.js.map} +1 -1
  39. package/public/assets/{markdown-preview-impl-yxQkrd2M.js → markdown-preview-impl-Wia3Ho1s.js} +2 -2
  40. package/public/assets/{markdown-preview-impl-yxQkrd2M.js.map → markdown-preview-impl-Wia3Ho1s.js.map} +1 -1
  41. package/public/assets/{memory-CJD8eekg.js → memory-B1v_zD-M.js} +2 -2
  42. package/public/assets/{memory-CJD8eekg.js.map → memory-B1v_zD-M.js.map} +1 -1
  43. package/public/assets/{messages-DLHMZbHZ.js → messages-BNOx3yK6.js} +2 -2
  44. package/public/assets/{messages-DLHMZbHZ.js.map → messages-BNOx3yK6.js.map} +1 -1
  45. package/public/assets/{orchestrators-DEOep7FX.js → orchestrators-DWFy4asl.js} +2 -2
  46. package/public/assets/{orchestrators-DEOep7FX.js.map → orchestrators-DWFy4asl.js.map} +1 -1
  47. package/public/assets/{overview-BgpfO6mt.js → overview-C_12cFPM.js} +2 -2
  48. package/public/assets/{overview-BgpfO6mt.js.map → overview-C_12cFPM.js.map} +1 -1
  49. package/public/assets/{pairs-8NVKHh96.js → pairs-DIc7gX83.js} +2 -2
  50. package/public/assets/{pairs-8NVKHh96.js.map → pairs-DIc7gX83.js.map} +1 -1
  51. package/public/assets/{projects-CZJW35C7.js → projects-DPjvJUzT.js} +2 -2
  52. package/public/assets/{projects-CZJW35C7.js.map → projects-DPjvJUzT.js.map} +1 -1
  53. package/public/assets/{prompt-settings-fqyLwUa-.js → prompt-settings-CtumNgG8.js} +2 -2
  54. package/public/assets/{prompt-settings-fqyLwUa-.js.map → prompt-settings-CtumNgG8.js.map} +1 -1
  55. package/public/assets/{registry-BKbLL4mO.js → registry-DzUhgMq9.js} +2 -2
  56. package/public/assets/{registry-BKbLL4mO.js.map → registry-DzUhgMq9.js.map} +1 -1
  57. package/public/assets/{security-BlTFTbq0.js → security-BJ3_u0uY.js} +2 -2
  58. package/public/assets/{security-BlTFTbq0.js.map → security-BJ3_u0uY.js.map} +1 -1
  59. package/public/assets/{select-Zd_qh0uU.js → select-AWvlctyQ.js} +2 -2
  60. package/public/assets/{select-Zd_qh0uU.js.map → select-AWvlctyQ.js.map} +1 -1
  61. package/public/assets/{settings-nHuSVRkL.js → settings-5ofqi-bN.js} +4 -4
  62. package/public/assets/settings-5ofqi-bN.js.map +1 -0
  63. package/public/assets/store-D5et8685.js +9 -0
  64. package/public/assets/store-D5et8685.js.map +1 -0
  65. package/public/assets/{tasks-C3JhQ8vB.js → tasks-BEE9bBf7.js} +2 -2
  66. package/public/assets/{tasks-C3JhQ8vB.js.map → tasks-BEE9bBf7.js.map} +1 -1
  67. package/public/assets/{teams-C9E9Yt39.js → teams-C27H_Gwo.js} +2 -2
  68. package/public/assets/{teams-C9E9Yt39.js.map → teams-C27H_Gwo.js.map} +1 -1
  69. package/public/assets/{terminal-viewer-impl-FP0JlsvS.js → terminal-viewer-impl-WD2TboA7.js} +2 -2
  70. package/public/assets/{terminal-viewer-impl-FP0JlsvS.js.map → terminal-viewer-impl-WD2TboA7.js.map} +1 -1
  71. package/public/assets/types-uVOXgvMT.js.map +1 -1
  72. package/public/assets/{work-queue-Ema7Hy3s.js → work-queue-CtmPK2hK.js} +2 -2
  73. package/public/assets/{work-queue-Ema7Hy3s.js.map → work-queue-CtmPK2hK.js.map} +1 -1
  74. package/public/assets/{workspaces-DI317fMk.js → workspaces-B26EKv_H.js} +2 -2
  75. package/public/assets/{workspaces-DI317fMk.js.map → workspaces-B26EKv_H.js.map} +1 -1
  76. package/public/index.html +3 -3
  77. package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
  78. package/src/agent-card-projection.ts +39 -9
  79. package/src/bus-outbox.ts +30 -0
  80. package/src/bus.ts +57 -1
  81. package/src/config-store.ts +2 -0
  82. package/src/config.ts +4 -0
  83. package/src/db/connection.ts +3 -1
  84. package/src/db/index.ts +1 -0
  85. package/src/db/maintenance-worker.ts +91 -0
  86. package/src/db/messages.ts +19 -4
  87. package/src/db/migrations.ts +7 -0
  88. package/src/db/retention.ts +130 -0
  89. package/src/db/schema.ts +5 -0
  90. package/src/db/stats.ts +22 -0
  91. package/src/event-loop-lag.ts +90 -0
  92. package/src/maintenance/jobs.ts +20 -6
  93. package/src/retention-config-store.ts +32 -0
  94. package/src/retention-config.ts +58 -0
  95. package/src/routes/messages.ts +13 -1
  96. package/src/routing-policy-store.ts +11 -1
  97. package/src/server.ts +2 -0
  98. package/src/services/send-message.ts +63 -21
  99. package/src/services/spawn-registration-watchdog.ts +74 -9
  100. package/src/settings/registry.ts +11 -0
  101. package/src/sse.ts +54 -9
  102. package/public/assets/settings-nHuSVRkL.js.map +0 -1
  103. package/public/assets/store-nXGxgz2Q.js +0 -9
  104. package/public/assets/store-nXGxgz2Q.js.map +0 -1
@@ -0,0 +1,90 @@
1
+ import {
2
+ EVENT_LOOP_LAG_LOG_MS,
3
+ EVENT_LOOP_LAG_SAMPLE_MS,
4
+ EVENT_LOOP_LAG_WARN_MS,
5
+ EVENT_LOOP_LAG_WINDOW_SAMPLES,
6
+ } from "./config";
7
+
8
+ export interface EventLoopLagSnapshot {
9
+ currentMs: number;
10
+ maxMs: number;
11
+ maxSinceLastReadMs: number;
12
+ sampleCount: number;
13
+ windowSamples: number;
14
+ sampleIntervalMs: number;
15
+ warnThresholdMs: number;
16
+ lastSampleAt?: number;
17
+ }
18
+
19
+ const samples = new Array<number>(EVENT_LOOP_LAG_WINDOW_SAMPLES).fill(0);
20
+ let sampleIndex = 0;
21
+ let sampleCount = 0;
22
+ let currentMs = 0;
23
+ let maxSinceLastReadMs = 0;
24
+ let lastSampleAt: number | undefined;
25
+ let timer: ReturnType<typeof setInterval> | undefined;
26
+ let lastTickAt = 0;
27
+
28
+ export function computeEventLoopLagMs(elapsedMs: number, expectedMs: number): number {
29
+ if (!Number.isFinite(elapsedMs) || !Number.isFinite(expectedMs)) return 0;
30
+ return Math.max(0, elapsedMs - expectedMs);
31
+ }
32
+
33
+ export function recordEventLoopLagSample(lagMs: number, sampledAt: number = Date.now(), logSpike = false): void {
34
+ const normalized = Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0;
35
+ currentMs = normalized;
36
+ maxSinceLastReadMs = Math.max(maxSinceLastReadMs, normalized);
37
+ lastSampleAt = sampledAt;
38
+ samples[sampleIndex] = normalized;
39
+ sampleIndex = (sampleIndex + 1) % samples.length;
40
+ sampleCount = Math.min(sampleCount + 1, samples.length);
41
+ if (logSpike && EVENT_LOOP_LAG_LOG_MS > 0 && normalized >= EVENT_LOOP_LAG_LOG_MS) {
42
+ console.warn(`[health] event-loop lag ${Math.round(normalized)}ms`);
43
+ }
44
+ }
45
+
46
+ export function getEventLoopLagSnapshot(): EventLoopLagSnapshot {
47
+ let maxMs = 0;
48
+ for (let i = 0; i < sampleCount; i += 1) maxMs = Math.max(maxMs, samples[i] ?? 0);
49
+ const maxSinceLastRead = maxSinceLastReadMs;
50
+ maxSinceLastReadMs = 0;
51
+ return {
52
+ currentMs,
53
+ maxMs,
54
+ maxSinceLastReadMs: maxSinceLastRead,
55
+ sampleCount,
56
+ windowSamples: EVENT_LOOP_LAG_WINDOW_SAMPLES,
57
+ sampleIntervalMs: EVENT_LOOP_LAG_SAMPLE_MS,
58
+ warnThresholdMs: EVENT_LOOP_LAG_WARN_MS,
59
+ lastSampleAt,
60
+ };
61
+ }
62
+
63
+ export function startEventLoopLagSampler(): void {
64
+ if (timer) return;
65
+ lastTickAt = performance.now();
66
+ timer = setInterval(() => {
67
+ const now = performance.now();
68
+ const lagMs = computeEventLoopLagMs(now - lastTickAt, EVENT_LOOP_LAG_SAMPLE_MS);
69
+ lastTickAt = now;
70
+ recordEventLoopLagSample(lagMs, Date.now(), true);
71
+ }, EVENT_LOOP_LAG_SAMPLE_MS);
72
+ timer.unref?.();
73
+ }
74
+
75
+ export function stopEventLoopLagSampler(): void {
76
+ if (!timer) return;
77
+ clearInterval(timer);
78
+ timer = undefined;
79
+ }
80
+
81
+ export function resetEventLoopLagForTests(): void {
82
+ stopEventLoopLagSampler();
83
+ samples.fill(0);
84
+ sampleIndex = 0;
85
+ sampleCount = 0;
86
+ currentMs = 0;
87
+ maxSinceLastReadMs = 0;
88
+ lastSampleAt = undefined;
89
+ lastTickAt = 0;
90
+ }
@@ -3,6 +3,7 @@ import { deriveAgentBusyHealth } from "../agent-busy-health";
3
3
  import { expireStaleBusAgents } from "../bus";
4
4
  import { pruneOutbox } from "../bus-outbox";
5
5
  import { expireCommands } from "../commands-db";
6
+ import { getRetentionConfig } from "../retention-config-store";
6
7
  import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
7
8
  import {
8
9
  createActivityEvent,
@@ -13,7 +14,7 @@ import {
13
14
  listStaleQueuedDirectTargets,
14
15
  pruneOrphanedSharedWorkspaces,
15
16
  pruneOfflineAgents,
16
- pruneOldMessagesAsync,
17
+ pruneRetentionBatchAsync,
17
18
  pruneProviderQuotaHistory,
18
19
  reapStaleAgents,
19
20
  reapStaleOrchestrators,
@@ -375,13 +376,26 @@ export const definitions: MaintenanceJobDefinition[] = [
375
376
  },
376
377
  {
377
378
  id: "old-message-prune",
378
- title: "Old message prune",
379
- description: "Delete messages outside the configured retention window.",
379
+ title: "Data retention prune",
380
+ description: "Delete old messages, activity events, and terminal spawn commands in bounded batches.",
380
381
  intervalMs: DAY_MS,
381
382
  runOnStart: false,
382
- handler(context) {
383
- const retentionDays = Number(process.env.RETENTION_DAYS) || 30;
384
- return pruneOldMessagesAsync(retentionDays * DAY_MS, { signal: context?.signal });
383
+ async handler(context) {
384
+ const retention = getRetentionConfig();
385
+ const now = Date.now();
386
+ const result = await pruneRetentionBatchAsync({
387
+ messageCutoff: now - retention.messageRetentionDays * DAY_MS,
388
+ activityCutoff: now - retention.activityEventRetentionDays * DAY_MS,
389
+ activityMaxRows: retention.activityEventMaxRows,
390
+ spawnCommandCutoff: now - retention.spawnCommandTerminalRetentionDays * DAY_MS,
391
+ batchSize: retention.pruneBatchSize,
392
+ }, { signal: context?.signal });
393
+ return {
394
+ prunedMessages: result.prunedMessages,
395
+ prunedActivityEvents: result.prunedActivityEvents,
396
+ prunedSpawnCommands: result.prunedSpawnCommands,
397
+ offThread: result.offThread,
398
+ };
385
399
  },
386
400
  },
387
401
  {
@@ -0,0 +1,32 @@
1
+ import { getConfig, setConfig } from "./config-store";
2
+ import {
3
+ RETENTION_CONFIG_DEFAULTS,
4
+ RETENTION_KEY,
5
+ RETENTION_NAMESPACE,
6
+ validateRetentionConfig,
7
+ type RetentionConfig,
8
+ } from "./retention-config";
9
+ import type { ConfigEntry } from "./types";
10
+
11
+ /** Instance-wide retention config, merged over defaults (always returns a usable value). */
12
+ export function getRetentionConfig(): RetentionConfig {
13
+ const entry = getConfig<Partial<RetentionConfig>>(RETENTION_NAMESPACE, RETENTION_KEY);
14
+ if (!entry) return { ...RETENTION_CONFIG_DEFAULTS };
15
+ return validateRetentionConfig({ ...RETENTION_CONFIG_DEFAULTS, ...entry.value });
16
+ }
17
+
18
+ export function getRetentionConfigEntry(): ConfigEntry<RetentionConfig> {
19
+ const entry = getConfig<RetentionConfig>(RETENTION_NAMESPACE, RETENTION_KEY);
20
+ return entry ?? {
21
+ namespace: RETENTION_NAMESPACE,
22
+ key: RETENTION_KEY,
23
+ value: { ...RETENTION_CONFIG_DEFAULTS },
24
+ version: 0,
25
+ updatedAt: "default",
26
+ updatedBy: "system",
27
+ };
28
+ }
29
+
30
+ export function setRetentionConfig(value: unknown, updatedBy?: string): ConfigEntry<RetentionConfig> {
31
+ return setConfig(RETENTION_NAMESPACE, RETENTION_KEY, value as RetentionConfig, updatedBy);
32
+ }
@@ -0,0 +1,58 @@
1
+ import { isRecord } from "agent-relay-sdk";
2
+ import { ValidationError } from "./db/connection.ts";
3
+
4
+ export const RETENTION_NAMESPACE = "retention";
5
+ export const RETENTION_KEY = "default";
6
+
7
+ export interface RetentionConfig {
8
+ messageRetentionDays: number;
9
+ activityEventRetentionDays: number;
10
+ activityEventMaxRows: number;
11
+ spawnCommandTerminalRetentionDays: number;
12
+ pruneBatchSize: number;
13
+ }
14
+
15
+ export const RETENTION_CONFIG_DEFAULTS: RetentionConfig = {
16
+ messageRetentionDays: 30,
17
+ activityEventRetentionDays: 14,
18
+ activityEventMaxRows: 100_000,
19
+ spawnCommandTerminalRetentionDays: 14,
20
+ pruneBatchSize: 10_000,
21
+ };
22
+
23
+ export const RETENTION_CONFIG_SCHEMA = {
24
+ type: "object",
25
+ properties: {
26
+ messageRetentionDays: { type: "integer", minimum: 1, maximum: 3650 },
27
+ activityEventRetentionDays: { type: "integer", minimum: 1, maximum: 3650 },
28
+ activityEventMaxRows: { type: "integer", minimum: 1, maximum: 10_000_000 },
29
+ spawnCommandTerminalRetentionDays: { type: "integer", minimum: 1, maximum: 3650 },
30
+ pruneBatchSize: { type: "integer", minimum: 1, maximum: 50_000 },
31
+ },
32
+ additionalProperties: false,
33
+ } as const;
34
+
35
+ function cleanInteger(value: unknown, field: string, fallback: number, min: number, max: number): number {
36
+ if (value === undefined || value === null) return fallback;
37
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) {
38
+ throw new ValidationError(`${field} must be an integer between ${min} and ${max}`);
39
+ }
40
+ return value;
41
+ }
42
+
43
+ export function validateRetentionConfig(value: unknown): RetentionConfig {
44
+ if (!isRecord(value)) throw new ValidationError("retention config value must be an object");
45
+ return {
46
+ messageRetentionDays: cleanInteger(value.messageRetentionDays, "messageRetentionDays", RETENTION_CONFIG_DEFAULTS.messageRetentionDays, 1, 3650),
47
+ activityEventRetentionDays: cleanInteger(value.activityEventRetentionDays, "activityEventRetentionDays", RETENTION_CONFIG_DEFAULTS.activityEventRetentionDays, 1, 3650),
48
+ activityEventMaxRows: cleanInteger(value.activityEventMaxRows, "activityEventMaxRows", RETENTION_CONFIG_DEFAULTS.activityEventMaxRows, 1, 10_000_000),
49
+ spawnCommandTerminalRetentionDays: cleanInteger(
50
+ value.spawnCommandTerminalRetentionDays,
51
+ "spawnCommandTerminalRetentionDays",
52
+ RETENTION_CONFIG_DEFAULTS.spawnCommandTerminalRetentionDays,
53
+ 1,
54
+ 3650,
55
+ ),
56
+ pruneBatchSize: cleanInteger(value.pruneBatchSize, "pruneBatchSize", RETENTION_CONFIG_DEFAULTS.pruneBatchSize, 1, 50_000),
57
+ };
58
+ }
@@ -7,7 +7,7 @@ import { emitMessageClaimed, emitMessageDeleted, emitMessageDeliveryUpdated, emi
7
7
  import { errMessage, isMechanicalMessageKind, isRecord } from "agent-relay-sdk";
8
8
  import { injectMemoryForMessageDelivery } from "../memory-service";
9
9
  import { authContextFromRequest } from "../services/auth-context";
10
- import { sendMessageService } from "../services/send-message";
10
+ import { sendMessagesBatchService, sendMessageService } from "../services/send-message";
11
11
  import { AmbiguousTargetError, ServiceAuthError } from "../services/errors";
12
12
  import { type AgentSessionGuard, type Message, type SendMessageInput } from "../types";
13
13
 
@@ -73,6 +73,7 @@ function isDirectTarget(to: string): boolean {
73
73
  }
74
74
 
75
75
  const VALID_MSG_KINDS = ["chat", "channel.event", "task", "pair", "control", "system", "session"];
76
+ const MAX_MESSAGE_BATCH = 100;
76
77
 
77
78
  export const getQueuedMessagesRoute: Handler = (req) => {
78
79
  const url = new URL(req.url);
@@ -149,6 +150,17 @@ export const postMessage: Handler = async (req) => {
149
150
  const parsed = await parseBody<unknown>(req);
150
151
  if (!parsed.ok) return error(parsed.error, parsed.status);
151
152
  try {
153
+ if (isRecord(parsed.body) && Array.isArray(parsed.body.messages)) {
154
+ if (parsed.body.messages.length === 0 || parsed.body.messages.length > MAX_MESSAGE_BATCH) {
155
+ throw new ValidationError(`messages must contain 1-${MAX_MESSAGE_BATCH} items`);
156
+ }
157
+ const inputs = parsed.body.messages.map((message) => normalizeMessageInput(message));
158
+ if (inputs.some((input) => input.kind !== "session")) {
159
+ throw new ValidationError("batch messages only supports kind=session");
160
+ }
161
+ const results = sendMessagesBatchService(inputs, authContextFromRequest(req));
162
+ return json({ messages: results.map((result) => result.message) }, 201);
163
+ }
152
164
  const input = normalizeMessageInput(parsed.body);
153
165
  if (!input.idempotencyKey) {
154
166
  input.idempotencyKey = cleanString(req.headers.get("Idempotency-Key") ?? undefined, "idempotencyKey", { max: 240 });
@@ -278,7 +278,7 @@ function normalizeProviderQuotaConfig(raw: unknown, provider: string): ProviderQ
278
278
  if (!isRecord(raw)) throw new ValidationError(`providerQuota.${provider} must be an object`);
279
279
  const cfg: ProviderQuotaRoutingConfig = {};
280
280
  if (raw.hardBackstopUtilization !== undefined) {
281
- cfg.hardBackstopUtilization = cleanFraction(raw.hardBackstopUtilization, `providerQuota.${provider}.hardBackstopUtilization`);
281
+ cfg.hardBackstopUtilization = cleanHardBackstopFraction(raw.hardBackstopUtilization);
282
282
  }
283
283
  if (raw.burnPenaltyUtilizationFloor !== undefined) {
284
284
  cfg.burnPenaltyUtilizationFloor = cleanFraction(raw.burnPenaltyUtilizationFloor, `providerQuota.${provider}.burnPenaltyUtilizationFloor`);
@@ -293,6 +293,16 @@ function cleanQuotaFraction(value: unknown, field: string): number {
293
293
  return value;
294
294
  }
295
295
 
296
+ function cleanHardBackstopFraction(value: unknown): number {
297
+ // hardBackstopUtilization=0 is an intentional contract: disable the hard backstop
298
+ // for that provider. Other invalid values normalize to the default instead.
299
+ if (value === 0) return 0;
300
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
301
+ return 0.99;
302
+ }
303
+ return value;
304
+ }
305
+
296
306
  function cleanFraction(value: unknown, field: string): number {
297
307
  if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
298
308
  throw new ValidationError(`${field} must be a number between 0 and 1`);
package/src/server.ts CHANGED
@@ -13,6 +13,7 @@ import { installNotificationSubscriptionRule } from "./notification-subscription
13
13
  import { installQuotaAdvisoryHandler } from "./quota-advisory";
14
14
  import { startPlanLiveBindings } from "./services/plan-live-bindings";
15
15
  import { startRelayScheduler } from "./services/scheduler";
16
+ import { startEventLoopLagSampler } from "./event-loop-lag";
16
17
  import { recordWorkspaceIssueActivity } from "./services/spawn-issue-activity";
17
18
  import { onWorkspaceCreate } from "./db";
18
19
  import { isTerminalSessionRequestAuthorized } from "./terminal-authz";
@@ -80,6 +81,7 @@ export function startServer(): void {
80
81
  getCompactionWatch().start();
81
82
  reconcileRelayVersion();
82
83
  startConnectorStatusPoller();
84
+ startEventLoopLagSampler();
83
85
  startMaintenanceScheduler();
84
86
  startRelayScheduler();
85
87
 
@@ -28,11 +28,13 @@ import {
28
28
  getMessage,
29
29
  listAgents,
30
30
  promoteLineageCapturedResponse,
31
+ sendMessagesBatchWithResult,
31
32
  sendMessageWithResult,
32
33
  channelObligationAnsweredBy,
34
+ type SendMessageResult as DbSendMessageResult,
33
35
  } from "../db";
34
36
  import { planSend, type DeliveryReceipt } from "../agent-ref";
35
- import { emitMessageQueued, emitNewMessage } from "../sse";
37
+ import { emitMessageQueued, emitNewMessages } from "../sse";
36
38
  import { getLifecycleManager } from "../lifecycle-manager";
37
39
  import { isComponentAuthorizedFor, isIntegrationAllowed } from "../security";
38
40
  import { AmbiguousTargetError, ServiceAuthError } from "./errors";
@@ -175,6 +177,42 @@ export function sendMessageService(
175
177
  ctx: AuthContext,
176
178
  opts: SendMessageOptions = {},
177
179
  ): SendMessageResult {
180
+ const { receipt, suppressedParent } = prepareSendMessage(input, ctx, opts);
181
+ if (suppressedParent) return { message: suppressedParent, created: false, receipt };
182
+ const result = sendMessageWithResult(input);
183
+ emitSendResults([result]);
184
+ return { message: result.message, created: result.created, receipt };
185
+ }
186
+
187
+ export function sendMessagesBatchService(
188
+ inputs: SendMessageInput[],
189
+ ctx: AuthContext,
190
+ opts: SendMessageOptions = {},
191
+ ): SendMessageResult[] {
192
+ const prepared = inputs.map((input) => ({ input, ...prepareSendMessage(input, ctx, opts) }));
193
+ const results: SendMessageResult[] = [];
194
+ const storeInputs: SendMessageInput[] = [];
195
+ const storeIndexes: number[] = [];
196
+ for (const item of prepared) {
197
+ if (item.suppressedParent) {
198
+ results.push({ message: item.suppressedParent, created: false, receipt: item.receipt });
199
+ continue;
200
+ }
201
+ results.push(null as unknown as SendMessageResult);
202
+ storeIndexes.push(results.length - 1);
203
+ storeInputs.push(item.input);
204
+ }
205
+ const stored = sendMessagesBatchWithResult(storeInputs);
206
+ for (let i = 0; i < stored.length; i++) {
207
+ const result = stored[i]!;
208
+ const outputIndex = storeIndexes[i]!;
209
+ results[outputIndex] = { message: result.message, created: result.created, receipt: prepared[outputIndex]!.receipt };
210
+ }
211
+ emitSendResults(stored);
212
+ return results;
213
+ }
214
+
215
+ function prepareSendMessage(input: SendMessageInput, ctx: AuthContext, opts: SendMessageOptions): { receipt: DeliveryReceipt; suppressedParent: Message | null } {
178
216
  const receipt = validateSendMessageAuthorization(input, ctx, opts);
179
217
  // Reply-obligation linking: some real answers are ordinary sends without replyTo/thread linkage.
180
218
  // Channel bridge replies have a single unambiguous open obligation case;
@@ -184,29 +222,33 @@ export function sendMessageService(
184
222
  const answered = channelObligationAnsweredBy(input.from, input.to);
185
223
  if (answered !== null) input.replyTo = answered;
186
224
  }
187
- const suppressedParent = suppressResultsOnlyAck(input);
188
- if (suppressedParent) return { message: suppressedParent, created: false, receipt };
189
- const result = sendMessageWithResult(input);
190
- if (result.created) {
191
- if (result.message.deliveryStatus === "queued") {
192
- emitMessageQueued(result.message);
193
- // Wake an on-demand managed policy that has no live agent attached yet.
194
- if (result.message.to?.startsWith("policy:")) {
195
- getLifecycleManager().onMessageForPolicy(result.message.to.slice("policy:".length));
225
+ return { receipt, suppressedParent: suppressResultsOnlyAck(input) };
226
+ }
227
+
228
+ function emitSendResults(results: DbSendMessageResult[]): void {
229
+ const liveMessages: Message[] = [];
230
+ for (const result of results) {
231
+ if (result.created) {
232
+ if (result.message.deliveryStatus === "queued") {
233
+ emitMessageQueued(result.message);
234
+ // Wake an on-demand managed policy that has no live agent attached yet.
235
+ if (result.message.to?.startsWith("policy:")) {
236
+ getLifecycleManager().onMessageForPolicy(result.message.to.slice("policy:".length));
237
+ }
238
+ } else {
239
+ liveMessages.push(result.message);
196
240
  }
197
- } else {
198
- emitNewMessage(result.message);
241
+ const promoted = promoteLineageCapturedResponse(result.message);
242
+ if (promoted?.created) liveMessages.push(promoted.message);
243
+ } else if (result.updated) {
244
+ // Session-step upsert (#parity): the row changed in place (same DB id). Re-emit so the
245
+ // dashboard's id-keyed merge replaces the existing step live — no duplicate row.
246
+ liveMessages.push(result.message);
247
+ const promoted = promoteLineageCapturedResponse(result.message);
248
+ if (promoted?.created) liveMessages.push(promoted.message);
199
249
  }
200
- const promoted = promoteLineageCapturedResponse(result.message);
201
- if (promoted?.created) emitNewMessage(promoted.message);
202
- } else if (result.updated) {
203
- // Session-step upsert (#parity): the row changed in place (same DB id). Re-emit so the
204
- // dashboard's id-keyed merge replaces the existing step live — no duplicate row.
205
- emitNewMessage(result.message);
206
- const promoted = promoteLineageCapturedResponse(result.message);
207
- if (promoted?.created) emitNewMessage(promoted.message);
208
250
  }
209
- return { message: result.message, created: result.created, receipt };
251
+ emitNewMessages(liveMessages);
210
252
  }
211
253
 
212
254
  function suppressResultsOnlyAck(input: SendMessageInput): Message | null {
@@ -20,12 +20,21 @@ interface SpawnCommandGroup {
20
20
  rows: SpawnCommandRow[];
21
21
  }
22
22
 
23
+ interface SpawnCandidateRow {
24
+ spawn_request_id: string;
25
+ latest_updated_at: number;
26
+ }
27
+
23
28
  interface SpawnRegistrationWatchdogResult extends Record<string, unknown> {
24
29
  failedSpawnCommandIds: string[];
25
30
  notifiedSpawnCommandIds: string[];
26
31
  silencedSpawnCommandIds: string[];
32
+ confirmedSpawnCommandIds: string[];
33
+ checkedSpawnGroups: number;
27
34
  }
28
35
 
36
+ const MAX_SPAWN_GROUPS_PER_RUN = 25;
37
+
29
38
  export function reapUnregisteredSpawnCommands(
30
39
  now = Date.now(),
31
40
  deadlineMs = SPAWN_REGISTRATION_DEADLINE_MS,
@@ -33,20 +42,40 @@ export function reapUnregisteredSpawnCommands(
33
42
  ): SpawnRegistrationWatchdogResult {
34
43
  const cutoff = now - deadlineMs;
35
44
  const notifyCutoff = now - notifyMaxAgeMs;
36
- const rows = getDb().query(`
37
- SELECT id, params, correlation_id, updated_at
38
- FROM commands
39
- WHERE type = 'agent.spawn'
40
- AND status = 'succeeded'
41
- ORDER BY updated_at ASC
42
- `).all() as SpawnCommandRow[];
45
+ const candidateRows = getDb().query(`
46
+ WITH spawn_groups AS (
47
+ SELECT
48
+ coalesce(nullif(json_extract(params, '$.spawnRequestId'), ''), correlation_id) AS spawn_request_id,
49
+ max(updated_at) AS latest_updated_at
50
+ FROM commands
51
+ WHERE type = 'agent.spawn'
52
+ AND status = 'succeeded'
53
+ AND json_valid(params)
54
+ AND coalesce(json_extract(params, '$.registrationConfirmed'), 0) != 1
55
+ AND coalesce(nullif(json_extract(params, '$.spawnedBy'), ''), '') != ''
56
+ GROUP BY coalesce(nullif(json_extract(params, '$.spawnRequestId'), ''), correlation_id)
57
+ )
58
+ SELECT spawn_request_id, latest_updated_at
59
+ FROM spawn_groups
60
+ WHERE spawn_request_id IS NOT NULL
61
+ AND latest_updated_at <= ?
62
+ ORDER BY latest_updated_at DESC
63
+ LIMIT ?
64
+ `).all(cutoff, MAX_SPAWN_GROUPS_PER_RUN) as SpawnCandidateRow[];
65
+ const rows = loadSpawnCommandRows(candidateRows.map((row) => row.spawn_request_id));
43
66
 
44
67
  const failedSpawnCommandIds: string[] = [];
45
68
  const notifiedSpawnCommandIds: string[] = [];
46
69
  const silencedSpawnCommandIds: string[] = [];
70
+ const confirmedSpawnCommandIds: string[] = [];
71
+ let checkedSpawnGroups = 0;
47
72
  for (const group of groupSpawnCommands(rows)) {
48
73
  if (group.latestUpdatedAt > cutoff) continue;
49
- if (hasEverRegisteredAgent(group.spawnRequestId)) continue;
74
+ checkedSpawnGroups++;
75
+ if (hasEverRegisteredAgent(group.spawnRequestId)) {
76
+ confirmedSpawnCommandIds.push(...markSpawnGroupRegistrationConfirmed(group, now));
77
+ continue;
78
+ }
50
79
 
51
80
  const reason = `spawned agent did not register within ${deadlineMs}ms`;
52
81
  const failedCommands = group.rows
@@ -70,7 +99,25 @@ export function reapUnregisteredSpawnCommands(
70
99
  }
71
100
  }
72
101
 
73
- return { failedSpawnCommandIds, notifiedSpawnCommandIds, silencedSpawnCommandIds };
102
+ return { failedSpawnCommandIds, notifiedSpawnCommandIds, silencedSpawnCommandIds, confirmedSpawnCommandIds, checkedSpawnGroups };
103
+ }
104
+
105
+ function loadSpawnCommandRows(spawnRequestIds: string[]): SpawnCommandRow[] {
106
+ if (spawnRequestIds.length === 0) return [];
107
+ const placeholders = spawnRequestIds.map(() => "?").join(",");
108
+ return getDb().query(`
109
+ SELECT id, params, correlation_id, updated_at
110
+ FROM commands
111
+ WHERE type = 'agent.spawn'
112
+ AND status = 'succeeded'
113
+ AND json_valid(params)
114
+ AND coalesce(json_extract(params, '$.registrationConfirmed'), 0) != 1
115
+ AND (
116
+ correlation_id IN (${placeholders})
117
+ OR json_extract(params, '$.spawnRequestId') IN (${placeholders})
118
+ )
119
+ ORDER BY updated_at ASC
120
+ `).all(...spawnRequestIds, ...spawnRequestIds) as SpawnCommandRow[];
74
121
  }
75
122
 
76
123
  function groupSpawnCommands(rows: SpawnCommandRow[]): SpawnCommandGroup[] {
@@ -121,6 +168,24 @@ function hasEverRegisteredAgent(spawnRequestId: string): boolean {
121
168
  return Boolean(registration);
122
169
  }
123
170
 
171
+ function markSpawnGroupRegistrationConfirmed(group: SpawnCommandGroup, now: number): string[] {
172
+ const confirmed: string[] = [];
173
+ const stmt = getDb().query(`
174
+ UPDATE commands
175
+ SET params = ?
176
+ WHERE id = ?
177
+ AND status = 'succeeded'
178
+ `);
179
+ for (const row of group.rows) {
180
+ const params = parseParams(row.params);
181
+ if (params.registrationConfirmed === true) continue;
182
+ params.registrationConfirmed = true;
183
+ params.registrationConfirmedAt = now;
184
+ if (stmt.run(JSON.stringify(params), row.id).changes > 0) confirmed.push(row.id);
185
+ }
186
+ return confirmed;
187
+ }
188
+
124
189
  function parseParams(raw: string): Record<string, unknown> {
125
190
  try {
126
191
  const parsed = JSON.parse(raw);
@@ -35,6 +35,7 @@ import {
35
35
  NOTIFICATIONS_SETTINGS_SCHEMA,
36
36
  parseNotificationSettingsKey,
37
37
  } from "../notification-settings";
38
+ import { RETENTION_CONFIG_DEFAULTS, RETENTION_CONFIG_SCHEMA, RETENTION_KEY, RETENTION_NAMESPACE } from "../retention-config";
38
39
 
39
40
  /** Fail-closed default scope — a descriptor that omits scope requires admin to
40
41
  * read and write (#385 owns the scope grammar; this layer only declares it). */
@@ -400,6 +401,16 @@ function seedBuiltInSettings(): void {
400
401
  display: { group: "Teams", label: "Team idle watchdog", description: "Coordinator nudge when a team has no busy members.", order: 45 },
401
402
  });
402
403
 
404
+ registerSetting({
405
+ namespace: RETENTION_NAMESPACE,
406
+ key: RETENTION_KEY,
407
+ schema: RETENTION_CONFIG_SCHEMA,
408
+ defaults: RETENTION_CONFIG_DEFAULTS,
409
+ scope: { read: "settings:read:system", write: "settings:write:system" },
410
+ tier: "server-runtime",
411
+ display: { group: "System", label: "Data retention", description: "Retention windows and batch size for high-churn Relay tables.", order: 46 },
412
+ });
413
+
403
414
  registerSetting({
404
415
  namespace: "provider",
405
416
  key: "chatCaptureMode",
package/src/sse.ts CHANGED
@@ -161,6 +161,10 @@ function fanout(event: RelayEvent): void {
161
161
  sendNewMessage(event.data as unknown as Message, event.seq);
162
162
  return;
163
163
  }
164
+ if (event.type === "message.batch") {
165
+ sendNewMessages(batchMessages(event.data), event.seq);
166
+ return;
167
+ }
164
168
  if (event.type === "task.updated" || event.type === "task.created" || event.type === "task.claimed" || event.type === "task.status") {
165
169
  sendTaskChanged(event.data as unknown as Task, event.type, event.seq);
166
170
  return;
@@ -172,28 +176,62 @@ function fanout(event: RelayEvent): void {
172
176
  }
173
177
 
174
178
  function sendNewMessage(msg: Message, seq?: number): void {
175
- const scopedBroadcastAgentIds = msg.to === "broadcast"
176
- ? new Set(matchingDeliveryAgents("broadcast", { from: msg.from, channel: msg.channel }).map((agent) => agent.id))
177
- : null;
178
- const chunk = sseFrame("message.new", msg, seq);
179
+ sendNewMessages([msg], seq);
180
+ }
181
+
182
+ function sendNewMessages(messages: Message[], seq?: number): void {
183
+ if (messages.length === 0) return;
184
+ const chunks = messages.map((msg, index) => [msg, sseFrame("message.new", msg, seq === undefined ? undefined : seq + index)] as const);
185
+ const broadcastScopes = new Map<number, Set<string>>();
186
+ for (const msg of messages) {
187
+ if (msg.to === "broadcast") {
188
+ broadcastScopes.set(msg.id, new Set(matchingDeliveryAgents("broadcast", { from: msg.from, channel: msg.channel }).map((agent) => agent.id)));
189
+ }
190
+ }
179
191
  const agentCache = new Map<string, AgentCard | null>();
180
192
  for (const conn of connections.values()) {
181
- if (!conn.agentId) {
193
+ for (const [msg, chunk] of chunks) {
194
+ if (!conn.agentId) {
195
+ sendEncoded(conn, chunk);
196
+ continue;
197
+ }
198
+ const scopedBroadcastAgentIds = broadcastScopes.get(msg.id) ?? null;
199
+ if (scopedBroadcastAgentIds && !scopedBroadcastAgentIds.has(conn.agentId)) continue;
200
+ if (!messageMatchesAgent(msg, conn.agentId, fanoutAgent(conn.agentId, agentCache))) continue;
182
201
  sendEncoded(conn, chunk);
183
- continue;
184
202
  }
185
- if (scopedBroadcastAgentIds && !scopedBroadcastAgentIds.has(conn.agentId)) continue;
186
- if (!messageMatchesAgent(msg, conn.agentId, fanoutAgent(conn.agentId, agentCache))) continue;
187
- sendEncoded(conn, chunk);
188
203
  }
189
204
  }
190
205
 
206
+ function batchMessages(data: Record<string, unknown>): Message[] {
207
+ const messages = (data as { messages?: unknown }).messages;
208
+ return Array.isArray(messages) ? messages as Message[] : [];
209
+ }
210
+
191
211
  export function emitNewMessage(msg: Message) {
192
212
  emitRelayEvent({ type: "message.new", source: msg.from, subject: String(msg.id), data: msg as unknown as Record<string, unknown> });
193
213
  const notification = notificationForMessage(msg);
194
214
  if (notification) emitNotificationCreated(notification);
195
215
  }
196
216
 
217
+ export function emitNewMessages(messages: Message[]) {
218
+ if (messages.length === 0) return;
219
+ if (messages.length === 1) {
220
+ emitNewMessage(messages[0]!);
221
+ return;
222
+ }
223
+ emitRelayEvent({
224
+ type: "message.batch",
225
+ source: messages[0]!.from,
226
+ subject: `${messages[0]!.id}:${messages.at(-1)!.id}`,
227
+ data: { messages } as unknown as Record<string, unknown>,
228
+ });
229
+ for (const msg of messages) {
230
+ const notification = notificationForMessage(msg);
231
+ if (notification) emitNotificationCreated(notification);
232
+ }
233
+ }
234
+
197
235
  interface RelayInjectionTimelineInput {
198
236
  agentId: string;
199
237
  category: RelayInjectionCategory;
@@ -328,6 +366,13 @@ function replayEventsToConnection(conn: Connection | undefined, plan: ReplayPlan
328
366
  }
329
367
 
330
368
  function sendReplayEvent(conn: Connection, event: RelayEvent, agentCache: Map<string, AgentCard | null>): void {
369
+ if (event.type === "message.batch") {
370
+ for (const [index, msg] of batchMessages(event.data).entries()) {
371
+ if (conn.agentId && !messageMatchesAgent(msg, conn.agentId, fanoutAgent(conn.agentId, agentCache))) continue;
372
+ send(conn, "message.new", msg, event.seq + index);
373
+ }
374
+ return;
375
+ }
331
376
  if (event.type === "message.new" && conn.agentId) {
332
377
  const msg = event.data as unknown as Message;
333
378
  if (!messageMatchesAgent(msg, conn.agentId, fanoutAgent(conn.agentId, agentCache))) return;