agent-relay-server 0.88.2 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +2 -2
- package/public/assets/display-CWMHll1J.js.map +1 -1
- package/runner/src/adapter.ts +4 -0
- package/src/agent-lifecycle-events.ts +74 -4
- package/src/branch-landed.ts +2 -2
- package/src/commands-db.ts +4 -1
- package/src/config.ts +7 -0
- package/src/db/index.ts +1 -0
- package/src/db/migrations.ts +52 -0
- package/src/db/scheduled-timers.ts +297 -0
- package/src/db/schema.ts +31 -0
- package/src/db/terminal-events.ts +17 -11
- package/src/index.ts +2 -0
- package/src/maintenance/teams.ts +7 -2
- package/src/mcp/index.ts +5 -0
- package/src/mcp/tools-messaging.ts +22 -2
- package/src/mcp/tools-scheduler.ts +135 -0
- package/src/notification-types.ts +2 -0
- package/src/quota-advisory.ts +27 -3
- package/src/services/scheduler.ts +303 -0
- package/src/services/send-message.ts +19 -18
- package/src/services/transient-agent-reaper.ts +1 -1
- package/src/team-idle-tracker.ts +6 -0
- package/src/token-db.ts +5 -5
package/src/maintenance/teams.ts
CHANGED
|
@@ -66,6 +66,7 @@ function detectSilentMembers(
|
|
|
66
66
|
watch.set(member.id, {
|
|
67
67
|
lastActiveAt: Math.max(member.lastSeen, existing?.lastActiveAt ?? 0),
|
|
68
68
|
pinged: recovered ? false : existing?.pinged ?? false,
|
|
69
|
+
epoch: member.epoch,
|
|
69
70
|
...(member.label ? { label: member.label } : {}),
|
|
70
71
|
});
|
|
71
72
|
}
|
|
@@ -77,7 +78,9 @@ function detectSilentMembers(
|
|
|
77
78
|
entry.pinged = true;
|
|
78
79
|
// A terminal event already fired — a #633 crash report (death) or a #645 clean terminus
|
|
79
80
|
// (transient reap after land / graceful shutdown). Either way this is not a silent death.
|
|
80
|
-
|
|
81
|
+
// #652: only suppress when the terminal event's epoch matches the member's observed epoch —
|
|
82
|
+
// a stale event from a prior epoch must not mask a real death in the current epoch.
|
|
83
|
+
if (getCrashReport(memberId) || hasTerminalEvent(memberId, entry.epoch)) continue;
|
|
81
84
|
if (!coordinatorId) continue;
|
|
82
85
|
const minutes = Math.floor(gapMs / 60_000);
|
|
83
86
|
notifyTarget(
|
|
@@ -124,6 +127,7 @@ function detectGoneCoordinator(
|
|
|
124
127
|
coordinatorId,
|
|
125
128
|
lastActiveAt: Math.max(coord.lastSeen, prior?.lastActiveAt ?? 0),
|
|
126
129
|
escalated: false, // alive again ⇒ re-arm the escalation
|
|
130
|
+
epoch: coord.epoch,
|
|
127
131
|
...(coord.label ? { label: coord.label } : {}),
|
|
128
132
|
});
|
|
129
133
|
return;
|
|
@@ -138,7 +142,8 @@ function detectGoneCoordinator(
|
|
|
138
142
|
|
|
139
143
|
// A terminal marker means a #645 clean terminus (graceful shutdown) or a #633 crash the operator was
|
|
140
144
|
// already surfaced on — not a SILENT death. Suppress.
|
|
141
|
-
|
|
145
|
+
// #652: only suppress when the terminal event's epoch matches the coordinator's observed epoch.
|
|
146
|
+
if (getCrashReport(coordinatorId) || hasTerminalEvent(coordinatorId, watch.epoch)) {
|
|
142
147
|
out.suppressedTeamIds.push(teamId);
|
|
143
148
|
return;
|
|
144
149
|
}
|
package/src/mcp/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type { JsonRpcId, JsonRpcRequest, McpAuthContext, ToolDefinition } from "
|
|
|
18
18
|
import { positiveIdOrUndefined, recordField, stringField } from "./validation";
|
|
19
19
|
import { IDENTITY_TOOLS, hasAnyScope, mcpAuthContext, relayCompactAndResume, relayRecall, relayWhoami } from "./identity";
|
|
20
20
|
import { MESSAGING_TOOLS, relayAgentCrashReport, relayAgentStatus, relayAttachArtifact, relayFindAgents, relayGetMessage, relayGetThread, relayReply, relaySendMessage, relayUploadArtifact } from "./tools-messaging";
|
|
21
|
+
import { SCHEDULER_TOOLS, relayCancelTimer, relayListTimers, relayScheduleTimer } from "./tools-scheduler";
|
|
21
22
|
import { SPAWN_TOOLS, relayMyAgents, relayShutdownAgent, relaySpawnAgent, relaySpawnTargets } from "./tools-spawn";
|
|
22
23
|
import { WORKSPACE_TOOLS, callerIsolatedWorkspace, relayWorkspaceList, relayWorkspaceMutation, relayWorkspaceStatus } from "./tools-workspace";
|
|
23
24
|
|
|
@@ -25,6 +26,7 @@ const MCP_PROTOCOL_VERSION = "2024-11-05";
|
|
|
25
26
|
const MCP_BODY_OVERHEAD_BYTES = 16 * 1024;
|
|
26
27
|
const TOOLS: ToolDefinition[] = [
|
|
27
28
|
...MESSAGING_TOOLS,
|
|
29
|
+
...SCHEDULER_TOOLS,
|
|
28
30
|
...IDENTITY_TOOLS,
|
|
29
31
|
...SPAWN_TOOLS,
|
|
30
32
|
AGENT_ACTION_TOOL,
|
|
@@ -121,6 +123,9 @@ async function callTool(auth: McpAuthContext, params: unknown): Promise<Record<s
|
|
|
121
123
|
else if (name === "relay_reply") result = relayReply(auth, args);
|
|
122
124
|
else if (name === "relay_get_message") result = relayGetMessage(args);
|
|
123
125
|
else if (name === "relay_get_thread") result = relayGetThread(args);
|
|
126
|
+
else if (name === "relay_schedule_timer") result = relayScheduleTimer(auth, args);
|
|
127
|
+
else if (name === "relay_list_timers") result = relayListTimers(auth, args);
|
|
128
|
+
else if (name === "relay_cancel_timer") result = relayCancelTimer(auth, args);
|
|
124
129
|
else if (name === "relay_upload_artifact") result = await relayUploadArtifact(auth, args);
|
|
125
130
|
else if (name === "relay_attach_artifact") result = relayAttachArtifact(auth, args);
|
|
126
131
|
else if (name === "relay_agent_status") result = relayAgentStatus(args);
|
|
@@ -30,11 +30,12 @@ import { McpAuthError, McpNotFoundError } from "../mcp-errors";
|
|
|
30
30
|
import { authContextFromMcp } from "../services/auth-context";
|
|
31
31
|
import { ServiceAuthError } from "../services/errors";
|
|
32
32
|
import { sendMessageService } from "../services/send-message";
|
|
33
|
+
import { scheduleTimer } from "../services/scheduler";
|
|
33
34
|
import { optionalEnum } from "../validation";
|
|
34
35
|
import { isRecord } from "agent-relay-sdk";
|
|
35
36
|
import type { ArtifactKind, ArtifactSensitivity, AttachmentRef, Message, SendMessageInput } from "../types";
|
|
36
37
|
import type { McpAuthContext, ToolDefinition } from "./types";
|
|
37
|
-
import { callerAgentId } from "./identity";
|
|
38
|
+
import { callerAgentId, hasScope } from "./identity";
|
|
38
39
|
import { enumField, optionalAttachments, optionalBoolean, optionalFutureTimestamp, optionalNonNegativeInt, optionalPositiveInt, optionalRecord, optionalString, positiveId, recordField, stringField } from "./validation";
|
|
39
40
|
|
|
40
41
|
const VALID_ARTIFACT_KINDS = ["image", "audio", "video", "document", "archive", "other"] as const;
|
|
@@ -57,6 +58,7 @@ export const MESSAGING_TOOLS: ToolDefinition[] = [
|
|
|
57
58
|
channel: { type: "string" },
|
|
58
59
|
claimable: { type: "boolean" },
|
|
59
60
|
idempotencyKey: { type: "string" },
|
|
61
|
+
deliveryTime: { type: "string", description: "Optional future ISO timestamp. When set, relay_send_message creates a one-shot durable timer instead of sending immediately. Requires schedule:write." },
|
|
60
62
|
attachments: { type: "array", items: { type: "object" } },
|
|
61
63
|
payload: { type: "object" },
|
|
62
64
|
meta: { type: "object" },
|
|
@@ -221,9 +223,27 @@ function runMcpSend(auth: McpAuthContext, input: SendMessageInput): Message & {
|
|
|
221
223
|
}
|
|
222
224
|
}
|
|
223
225
|
|
|
224
|
-
export function relaySendMessage(auth: McpAuthContext, args: Record<string, unknown>): Message & { delivery: DeliveryReceipt } {
|
|
226
|
+
export function relaySendMessage(auth: McpAuthContext, args: Record<string, unknown>): (Message & { delivery: DeliveryReceipt }) | Record<string, unknown> {
|
|
225
227
|
const attachments = optionalAttachments(args.attachments);
|
|
226
228
|
const payload = payloadWithAttachments(optionalRecord(args.payload, "payload"), attachments);
|
|
229
|
+
const deliveryTime = optionalString(args.deliveryTime, "deliveryTime", 80);
|
|
230
|
+
if (deliveryTime) {
|
|
231
|
+
if (!hasScope(auth, "schedule:write")) throw new McpAuthError("deliveryTime requires schedule:write");
|
|
232
|
+
try {
|
|
233
|
+
const timer = scheduleTimer({
|
|
234
|
+
target: stringField(args.to, "to", { required: true, max: 200 }),
|
|
235
|
+
at: deliveryTime,
|
|
236
|
+
body: stringField(args.body, "body", { required: true, maxBytes: MAX_BODY_BYTES }),
|
|
237
|
+
subject: optionalString(args.subject, "subject", 200),
|
|
238
|
+
payload,
|
|
239
|
+
idempotencyKey: optionalString(args.idempotencyKey, "idempotencyKey", 240),
|
|
240
|
+
}, authContextFromMcp({ kind: auth.kind, actor: auth.actor, scopes: auth.scopes, component: auth.component }));
|
|
241
|
+
return { scheduled: true, timer };
|
|
242
|
+
} catch (e) {
|
|
243
|
+
if (e instanceof ServiceAuthError) throw new McpAuthError(e.message);
|
|
244
|
+
throw e;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
227
247
|
const input: SendMessageInput = {
|
|
228
248
|
from: optionalString(args.from, "from", 200) ?? "",
|
|
229
249
|
to: stringField(args.to, "to", { required: true, max: 200 }),
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { McpAuthError } from "../mcp-errors";
|
|
2
|
+
import { authContextFromMcp } from "../services/auth-context";
|
|
3
|
+
import { ServiceAuthError } from "../services/errors";
|
|
4
|
+
import { cancelTimer, listTimers, scheduleTimer, type ScheduleTimerInput } from "../services/scheduler";
|
|
5
|
+
import type { ScheduledTimer } from "../db";
|
|
6
|
+
import type { McpAuthContext, ToolDefinition } from "./types";
|
|
7
|
+
import { optionalBoolean, optionalPositiveInt, optionalRecord, optionalString, stringField } from "./validation";
|
|
8
|
+
|
|
9
|
+
export const SCHEDULER_TOOLS: ToolDefinition[] = [
|
|
10
|
+
{
|
|
11
|
+
name: "relay_schedule_timer",
|
|
12
|
+
description: "Create a durable Relay timer that wakes a target by enqueueing a normal Relay message when due. Supports one-shot afterMs/at and fixed recurring everyMs. RRULE is intentionally deferred.",
|
|
13
|
+
requiredScopes: ["schedule:write"],
|
|
14
|
+
inputSchema: {
|
|
15
|
+
type: "object",
|
|
16
|
+
properties: {
|
|
17
|
+
target: { type: "string", description: "Defaults to me. Supports agent id, policy:name, label:x, channel targets, and other normal Relay targets." },
|
|
18
|
+
afterMs: { type: "integer", minimum: 1 },
|
|
19
|
+
at: { type: "string", description: "Future ISO timestamp." },
|
|
20
|
+
everyMs: { type: "integer", minimum: 1 },
|
|
21
|
+
subject: { type: "string" },
|
|
22
|
+
body: { type: "string" },
|
|
23
|
+
payload: { type: "object" },
|
|
24
|
+
maxRuns: { type: "integer", minimum: 1 },
|
|
25
|
+
expiresAt: { oneOf: [{ type: "string" }, { type: "integer", minimum: 1 }] },
|
|
26
|
+
idempotencyKey: { type: "string" },
|
|
27
|
+
},
|
|
28
|
+
required: ["body"],
|
|
29
|
+
additionalProperties: false,
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: "relay_list_timers",
|
|
34
|
+
description: "List your durable Relay timers. Admin callers may pass createdBy to inspect another creator.",
|
|
35
|
+
requiredScopes: ["schedule:write"],
|
|
36
|
+
inputSchema: {
|
|
37
|
+
type: "object",
|
|
38
|
+
properties: {
|
|
39
|
+
includeDisabled: { type: "boolean" },
|
|
40
|
+
limit: { type: "integer", minimum: 1, maximum: 500 },
|
|
41
|
+
createdBy: { type: "string" },
|
|
42
|
+
},
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
name: "relay_cancel_timer",
|
|
48
|
+
description: "Cancel one of your durable Relay timers. Admin callers may cancel any timer.",
|
|
49
|
+
requiredScopes: ["schedule:write"],
|
|
50
|
+
inputSchema: {
|
|
51
|
+
type: "object",
|
|
52
|
+
properties: {
|
|
53
|
+
timerId: { type: "string" },
|
|
54
|
+
},
|
|
55
|
+
required: ["timerId"],
|
|
56
|
+
additionalProperties: false,
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
export function relayScheduleTimer(auth: McpAuthContext, args: Record<string, unknown>): Record<string, unknown> {
|
|
62
|
+
return withAuthErrors(() => ({
|
|
63
|
+
timer: serializeTimer(scheduleTimer(cleanScheduleInput(args), authContextFromMcp(authContextPayload(auth)))),
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function relayListTimers(auth: McpAuthContext, args: Record<string, unknown>): Record<string, unknown> {
|
|
68
|
+
return withAuthErrors(() => {
|
|
69
|
+
const timers = listTimers(authContextFromMcp(authContextPayload(auth)), {
|
|
70
|
+
includeDisabled: optionalBoolean(args.includeDisabled, "includeDisabled"),
|
|
71
|
+
limit: optionalPositiveInt(args.limit, "limit"),
|
|
72
|
+
createdBy: optionalString(args.createdBy, "createdBy", 200),
|
|
73
|
+
});
|
|
74
|
+
return { timers: timers.map(serializeTimer), count: timers.length };
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function relayCancelTimer(auth: McpAuthContext, args: Record<string, unknown>): Record<string, unknown> {
|
|
79
|
+
return withAuthErrors(() => ({
|
|
80
|
+
timer: serializeTimer(cancelTimer(stringField(args.timerId, "timerId", { required: true, max: 120 }), authContextFromMcp(authContextPayload(auth)))),
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function cleanScheduleInput(args: Record<string, unknown>): ScheduleTimerInput {
|
|
85
|
+
return {
|
|
86
|
+
target: optionalString(args.target, "target", 200),
|
|
87
|
+
afterMs: optionalPositiveInt(args.afterMs, "afterMs"),
|
|
88
|
+
at: optionalString(args.at, "at", 80),
|
|
89
|
+
everyMs: optionalPositiveInt(args.everyMs, "everyMs"),
|
|
90
|
+
subject: optionalString(args.subject, "subject", 200),
|
|
91
|
+
body: stringField(args.body, "body", { required: true }),
|
|
92
|
+
payload: optionalRecord(args.payload, "payload"),
|
|
93
|
+
maxRuns: optionalPositiveInt(args.maxRuns, "maxRuns"),
|
|
94
|
+
expiresAt: typeof args.expiresAt === "number" ? args.expiresAt : optionalString(args.expiresAt, "expiresAt", 80),
|
|
95
|
+
idempotencyKey: optionalString(args.idempotencyKey, "idempotencyKey", 240),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function authContextPayload(auth: McpAuthContext): Parameters<typeof authContextFromMcp>[0] {
|
|
100
|
+
return { kind: auth.kind, actor: auth.actor, scopes: auth.scopes, component: auth.component };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function serializeTimer(timer: ScheduledTimer): Record<string, unknown> {
|
|
104
|
+
return {
|
|
105
|
+
id: timer.id,
|
|
106
|
+
target: timer.target,
|
|
107
|
+
createdBy: timer.createdBy,
|
|
108
|
+
createdAt: timer.createdAt,
|
|
109
|
+
nextRunAt: timer.nextRunAt,
|
|
110
|
+
scheduleKind: timer.scheduleKind,
|
|
111
|
+
scheduleDef: timer.scheduleDef,
|
|
112
|
+
subject: timer.subject,
|
|
113
|
+
body: timer.body,
|
|
114
|
+
payload: timer.payload,
|
|
115
|
+
enabled: timer.enabled,
|
|
116
|
+
runCount: timer.runCount,
|
|
117
|
+
maxRuns: timer.maxRuns,
|
|
118
|
+
expiresAt: timer.expiresAt,
|
|
119
|
+
lastRunAt: timer.lastRunAt,
|
|
120
|
+
lastError: timer.lastError,
|
|
121
|
+
consecutiveFailures: timer.consecutiveFailures,
|
|
122
|
+
idempotencyKey: timer.idempotencyKey,
|
|
123
|
+
correlationId: timer.correlationId,
|
|
124
|
+
lastMessageId: timer.lastMessageId,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function withAuthErrors<T>(fn: () => T): T {
|
|
129
|
+
try {
|
|
130
|
+
return fn();
|
|
131
|
+
} catch (error) {
|
|
132
|
+
if (error instanceof ServiceAuthError) throw new McpAuthError(error.message);
|
|
133
|
+
throw error;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -4,6 +4,7 @@ export type NotificationType =
|
|
|
4
4
|
| "agent.ready"
|
|
5
5
|
| "agent.exited"
|
|
6
6
|
| "agent.spawn_failed"
|
|
7
|
+
| "agent.spawn_recovered"
|
|
7
8
|
| "agent.context_threshold"
|
|
8
9
|
| "agent.stuck_busy"
|
|
9
10
|
| "agent.quota_threshold"
|
|
@@ -59,6 +60,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
|
|
|
59
60
|
"agent.ready": null,
|
|
60
61
|
"agent.exited": null,
|
|
61
62
|
"agent.spawn_failed": null,
|
|
63
|
+
"agent.spawn_recovered": null,
|
|
62
64
|
"agent.context_threshold": 15 * MINUTES,
|
|
63
65
|
"agent.stuck_busy": null,
|
|
64
66
|
"agent.quota_threshold": 15 * MINUTES,
|
package/src/quota-advisory.ts
CHANGED
|
@@ -7,15 +7,17 @@ import { parseJson } from "./utils";
|
|
|
7
7
|
import { setProviderQuotaAdvisoryHandler, type ProviderQuotaAdvisoryInput } from "./db/provider-quotas.ts";
|
|
8
8
|
import type { QuotaState } from "./types";
|
|
9
9
|
|
|
10
|
+
type QuotaAdvisoryStage = "warn" | "critical" | "reset";
|
|
11
|
+
|
|
10
12
|
interface QuotaAdvisoryWindowState {
|
|
11
13
|
lastQuotaAdvisoryBand?: number;
|
|
14
|
+
lastQuotaAdvisoryStage?: QuotaAdvisoryStage;
|
|
12
15
|
lastQuotaAdvisoryAt?: number;
|
|
13
16
|
lastRemainingPercent?: number;
|
|
14
17
|
lastResetsAt?: number;
|
|
15
18
|
}
|
|
16
19
|
|
|
17
20
|
type QuotaAdvisoryState = Record<string, QuotaAdvisoryWindowState>;
|
|
18
|
-
type QuotaAdvisoryStage = "warn" | "critical" | "reset";
|
|
19
21
|
|
|
20
22
|
type AgentRecipientRow = {
|
|
21
23
|
id: string;
|
|
@@ -45,7 +47,7 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
|
|
|
45
47
|
const remaining = quotaRemaining(window.utilization);
|
|
46
48
|
const remainingPercent = percent(remaining);
|
|
47
49
|
const previous = nextState[window.name];
|
|
48
|
-
const reset = previous
|
|
50
|
+
const reset = hasLastQuotaAdvisory(previous) && isQuotaReset(remaining, previousWindow, window, config.resetRemaining);
|
|
49
51
|
if (reset) {
|
|
50
52
|
notifyQuotaAdvisory({
|
|
51
53
|
provider: input.provider,
|
|
@@ -69,8 +71,8 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
|
|
|
69
71
|
}
|
|
70
72
|
|
|
71
73
|
const band = quotaRemainingBand(remaining, config.bandSize);
|
|
72
|
-
if (previous?.lastQuotaAdvisoryBand === band) continue;
|
|
73
74
|
const stage: QuotaAdvisoryStage = remaining - config.criticalRemaining <= QUOTA_THRESHOLD_EPSILON ? "critical" : "warn";
|
|
75
|
+
if (lastQuotaAdvisoryStage(previous, config.criticalRemaining) === stage) continue;
|
|
74
76
|
notifyQuotaAdvisory({
|
|
75
77
|
provider: input.provider,
|
|
76
78
|
accountKey: input.accountKey,
|
|
@@ -85,6 +87,7 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
|
|
|
85
87
|
});
|
|
86
88
|
nextState[window.name] = {
|
|
87
89
|
lastQuotaAdvisoryBand: band,
|
|
90
|
+
lastQuotaAdvisoryStage: stage,
|
|
88
91
|
lastQuotaAdvisoryAt: input.now,
|
|
89
92
|
lastRemainingPercent: remainingPercent,
|
|
90
93
|
...(window.resetsAt !== undefined ? { lastResetsAt: window.resetsAt } : {}),
|
|
@@ -94,6 +97,27 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
|
|
|
94
97
|
return JSON.stringify(nextState);
|
|
95
98
|
}
|
|
96
99
|
|
|
100
|
+
function hasLastQuotaAdvisory(previous: QuotaAdvisoryWindowState | undefined): boolean {
|
|
101
|
+
return previous?.lastQuotaAdvisoryStage !== undefined || previous?.lastQuotaAdvisoryBand !== undefined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function lastQuotaAdvisoryStage(
|
|
105
|
+
previous: QuotaAdvisoryWindowState | undefined,
|
|
106
|
+
criticalRemaining: number,
|
|
107
|
+
): QuotaAdvisoryStage | undefined {
|
|
108
|
+
if (!previous) return undefined;
|
|
109
|
+
if (previous.lastQuotaAdvisoryStage === "warn" || previous.lastQuotaAdvisoryStage === "critical") {
|
|
110
|
+
return previous.lastQuotaAdvisoryStage;
|
|
111
|
+
}
|
|
112
|
+
if (typeof previous.lastRemainingPercent === "number") {
|
|
113
|
+
return previous.lastRemainingPercent <= percent(criticalRemaining) ? "critical" : "warn";
|
|
114
|
+
}
|
|
115
|
+
if (typeof previous.lastQuotaAdvisoryBand === "number") {
|
|
116
|
+
return previous.lastQuotaAdvisoryBand <= percent(criticalRemaining) ? "critical" : "warn";
|
|
117
|
+
}
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
|
|
97
121
|
function notifyQuotaAdvisory(input: {
|
|
98
122
|
provider: string;
|
|
99
123
|
accountKey: string;
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { errMessage } from "agent-relay-sdk";
|
|
3
|
+
import { hasScope } from "agent-relay-sdk/types";
|
|
4
|
+
import {
|
|
5
|
+
MAX_BODY_BYTES,
|
|
6
|
+
SCHEDULER_CLAIM_LEASE_MS,
|
|
7
|
+
SCHEDULER_MAX_ACTIVE_TIMERS,
|
|
8
|
+
SCHEDULER_MAX_FAILURES,
|
|
9
|
+
SCHEDULER_MAX_PAYLOAD_BYTES,
|
|
10
|
+
SCHEDULER_MIN_RECURRING_INTERVAL_MS,
|
|
11
|
+
SCHEDULER_RETRY_MS,
|
|
12
|
+
SCHEDULER_TICK_MS,
|
|
13
|
+
} from "../config";
|
|
14
|
+
import {
|
|
15
|
+
ValidationError,
|
|
16
|
+
cancelScheduledTimer,
|
|
17
|
+
claimScheduledTimer,
|
|
18
|
+
completeScheduledTimerRun,
|
|
19
|
+
countActiveScheduledTimers,
|
|
20
|
+
disableScheduledTimer,
|
|
21
|
+
getScheduledTimer,
|
|
22
|
+
insertScheduledTimer,
|
|
23
|
+
listDueScheduledTimerIds,
|
|
24
|
+
listScheduledTimers,
|
|
25
|
+
recordScheduledTimerFailure,
|
|
26
|
+
type ScheduledTimer,
|
|
27
|
+
type TimerScheduleDef,
|
|
28
|
+
type TimerScheduleKind,
|
|
29
|
+
} from "../db";
|
|
30
|
+
import type { ComponentToken, SendMessageInput } from "../types";
|
|
31
|
+
import { ServiceAuthError } from "./errors";
|
|
32
|
+
import type { AuthContext } from "./auth-context";
|
|
33
|
+
import { sendMessageService, validateSendMessageAuthorization } from "./send-message";
|
|
34
|
+
|
|
35
|
+
export interface ScheduleTimerInput {
|
|
36
|
+
target?: string;
|
|
37
|
+
afterMs?: number;
|
|
38
|
+
at?: string;
|
|
39
|
+
everyMs?: number;
|
|
40
|
+
rrule?: string;
|
|
41
|
+
timezone?: string;
|
|
42
|
+
subject?: string;
|
|
43
|
+
body: string;
|
|
44
|
+
payload?: Record<string, unknown>;
|
|
45
|
+
maxRuns?: number;
|
|
46
|
+
expiresAt?: string | number;
|
|
47
|
+
idempotencyKey?: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface RunDueTimersResult {
|
|
51
|
+
claimed: number;
|
|
52
|
+
fired: number;
|
|
53
|
+
failed: number;
|
|
54
|
+
disabled: number;
|
|
55
|
+
messageIds: number[];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
let schedulerTimer: Timer | undefined;
|
|
59
|
+
|
|
60
|
+
export function startRelayScheduler(): void {
|
|
61
|
+
if (schedulerTimer) return;
|
|
62
|
+
void runDueScheduledTimers().catch((error) => console.warn(`relay scheduler tick failed: ${errMessage(error)}`));
|
|
63
|
+
schedulerTimer = setInterval(() => {
|
|
64
|
+
void runDueScheduledTimers().catch((error) => console.warn(`relay scheduler tick failed: ${errMessage(error)}`));
|
|
65
|
+
}, SCHEDULER_TICK_MS);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function stopRelaySchedulerForTests(): void {
|
|
69
|
+
if (schedulerTimer) clearInterval(schedulerTimer);
|
|
70
|
+
schedulerTimer = undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function scheduleTimer(input: ScheduleTimerInput, ctx: AuthContext): ScheduledTimer {
|
|
74
|
+
const now = Date.now();
|
|
75
|
+
const schedule = normalizeSchedule(input, now);
|
|
76
|
+
const target = normalizeTarget(input.target, ctx);
|
|
77
|
+
const createdBy = ctx.callerAgentId ?? ctx.actor.id;
|
|
78
|
+
const deliveryFrom = ctx.callerAgentId ?? "system";
|
|
79
|
+
const body = cleanBody(input.body);
|
|
80
|
+
const payload = cleanPayload(input.payload);
|
|
81
|
+
const maxRuns = cleanPositiveInt(input.maxRuns, "maxRuns");
|
|
82
|
+
const expiresAt = cleanExpiresAt(input.expiresAt, now);
|
|
83
|
+
const idempotencyKey = cleanOptionalString(input.idempotencyKey, "idempotencyKey", 240);
|
|
84
|
+
|
|
85
|
+
if (schedule.kind === "interval") {
|
|
86
|
+
if ((schedule.def.everyMs ?? 0) < SCHEDULER_MIN_RECURRING_INTERVAL_MS) {
|
|
87
|
+
throw new ValidationError(`everyMs must be at least ${SCHEDULER_MIN_RECURRING_INTERVAL_MS}`);
|
|
88
|
+
}
|
|
89
|
+
if (!maxRuns && !expiresAt && !canCreatePersistentTimer(ctx)) {
|
|
90
|
+
throw new ServiceAuthError("recurring timers require maxRuns, expiresAt, or schedule:persistent");
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (expiresAt !== undefined && expiresAt <= schedule.nextRunAt) {
|
|
94
|
+
throw new ValidationError("expiresAt must be after the first run");
|
|
95
|
+
}
|
|
96
|
+
if (countActiveScheduledTimers(createdBy) >= SCHEDULER_MAX_ACTIVE_TIMERS) {
|
|
97
|
+
throw new ServiceAuthError(`active timer quota exceeded (${SCHEDULER_MAX_ACTIVE_TIMERS})`);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
validateSendMessageAuthorization({
|
|
101
|
+
from: deliveryFrom,
|
|
102
|
+
to: target,
|
|
103
|
+
body,
|
|
104
|
+
subject: cleanOptionalString(input.subject, "subject", 200),
|
|
105
|
+
payload,
|
|
106
|
+
idempotencyKey: idempotencyKey ? `${idempotencyKey}:probe` : undefined,
|
|
107
|
+
}, ctx);
|
|
108
|
+
|
|
109
|
+
return insertScheduledTimer({
|
|
110
|
+
target,
|
|
111
|
+
createdBy,
|
|
112
|
+
createdByKind: ctx.actor.kind,
|
|
113
|
+
createdByScopes: ctx.scopes,
|
|
114
|
+
createdByConstraints: ctx.constraints,
|
|
115
|
+
deliveryFrom,
|
|
116
|
+
nextRunAt: schedule.nextRunAt,
|
|
117
|
+
scheduleKind: schedule.kind,
|
|
118
|
+
scheduleDef: schedule.def,
|
|
119
|
+
subject: cleanOptionalString(input.subject, "subject", 200),
|
|
120
|
+
body,
|
|
121
|
+
payload,
|
|
122
|
+
maxRuns,
|
|
123
|
+
expiresAt,
|
|
124
|
+
idempotencyKey,
|
|
125
|
+
}, now);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function listTimers(ctx: AuthContext, input: { includeDisabled?: boolean; limit?: number; createdBy?: string } = {}): ScheduledTimer[] {
|
|
129
|
+
const admin = canAdministerTimers(ctx);
|
|
130
|
+
const createdBy = admin ? input.createdBy : (ctx.callerAgentId ?? ctx.actor.id);
|
|
131
|
+
return listScheduledTimers({
|
|
132
|
+
createdBy,
|
|
133
|
+
includeDisabled: input.includeDisabled,
|
|
134
|
+
limit: input.limit,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function cancelTimer(id: string, ctx: AuthContext): ScheduledTimer {
|
|
139
|
+
const timer = getScheduledTimer(id);
|
|
140
|
+
if (!timer) throw new ValidationError(`timer ${id} not found`);
|
|
141
|
+
if (!canAdministerTimers(ctx) && timer.createdBy !== (ctx.callerAgentId ?? ctx.actor.id)) {
|
|
142
|
+
throw new ServiceAuthError("cannot cancel a timer created by another caller");
|
|
143
|
+
}
|
|
144
|
+
return cancelScheduledTimer(id) ?? timer;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export async function runDueScheduledTimers(input: { now?: number; limit?: number; owner?: string } = {}): Promise<RunDueTimersResult> {
|
|
148
|
+
const now = input.now ?? Date.now();
|
|
149
|
+
const owner = input.owner ?? `${process.pid}-${randomUUID()}`;
|
|
150
|
+
const ids = listDueScheduledTimerIds(now, input.limit ?? 25);
|
|
151
|
+
const result: RunDueTimersResult = { claimed: 0, fired: 0, failed: 0, disabled: 0, messageIds: [] };
|
|
152
|
+
for (const id of ids) {
|
|
153
|
+
const timer = claimScheduledTimer(id, owner, now, SCHEDULER_CLAIM_LEASE_MS);
|
|
154
|
+
if (!timer) continue;
|
|
155
|
+
result.claimed += 1;
|
|
156
|
+
const fired = fireClaimedTimer(timer, now);
|
|
157
|
+
if (fired.status === "fired") {
|
|
158
|
+
result.fired += 1;
|
|
159
|
+
result.messageIds.push(fired.messageId);
|
|
160
|
+
} else if (fired.status === "disabled") {
|
|
161
|
+
result.disabled += 1;
|
|
162
|
+
} else {
|
|
163
|
+
result.failed += 1;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return result;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function fireClaimedTimer(timer: ScheduledTimer, now: number): { status: "fired"; messageId: number } | { status: "failed" | "disabled" } {
|
|
170
|
+
if (timer.expiresAt !== undefined && timer.expiresAt <= now) {
|
|
171
|
+
disableScheduledTimer(timer.id, "timer expired before delivery");
|
|
172
|
+
return { status: "disabled" };
|
|
173
|
+
}
|
|
174
|
+
try {
|
|
175
|
+
const runNumber = timer.runCount + 1;
|
|
176
|
+
const result = sendMessageService({
|
|
177
|
+
from: timer.deliveryFrom,
|
|
178
|
+
to: timer.target,
|
|
179
|
+
subject: timer.subject,
|
|
180
|
+
body: timer.body,
|
|
181
|
+
payload: {
|
|
182
|
+
...timer.payload,
|
|
183
|
+
scheduler: {
|
|
184
|
+
timerId: timer.id,
|
|
185
|
+
correlationId: timer.correlationId,
|
|
186
|
+
run: runNumber,
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
meta: {
|
|
190
|
+
scheduler: {
|
|
191
|
+
timerId: timer.id,
|
|
192
|
+
correlationId: timer.correlationId,
|
|
193
|
+
run: runNumber,
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
idempotencyKey: timerRunIdempotencyKey(timer, runNumber),
|
|
197
|
+
} satisfies SendMessageInput, persistedAuthContext(timer));
|
|
198
|
+
completeScheduledTimerRun({ timer, messageId: result.message.id, now });
|
|
199
|
+
return { status: "fired", messageId: result.message.id };
|
|
200
|
+
} catch (error) {
|
|
201
|
+
const updated = recordScheduledTimerFailure({
|
|
202
|
+
id: timer.id,
|
|
203
|
+
error: errMessage(error),
|
|
204
|
+
now,
|
|
205
|
+
retryMs: SCHEDULER_RETRY_MS,
|
|
206
|
+
maxFailures: SCHEDULER_MAX_FAILURES,
|
|
207
|
+
});
|
|
208
|
+
return updated?.enabled ? { status: "failed" } : { status: "disabled" };
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function timerRunIdempotencyKey(timer: ScheduledTimer, runNumber: number): string {
|
|
213
|
+
const base = timer.idempotencyKey ?? `scheduler:${timer.id}`;
|
|
214
|
+
return `${base}:run:${runNumber}`;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function persistedAuthContext(timer: ScheduledTimer): AuthContext {
|
|
218
|
+
const component: ComponentToken = {
|
|
219
|
+
sub: timer.createdBy,
|
|
220
|
+
role: timer.createdByKind,
|
|
221
|
+
scope: timer.createdByScopes,
|
|
222
|
+
constraints: timer.createdByConstraints,
|
|
223
|
+
iat: Math.floor(timer.createdAt / 1000),
|
|
224
|
+
};
|
|
225
|
+
return {
|
|
226
|
+
actor: { id: timer.createdBy, kind: "component" },
|
|
227
|
+
scopes: timer.createdByScopes,
|
|
228
|
+
constraints: timer.createdByConstraints,
|
|
229
|
+
callerAgentId: timer.deliveryFrom === "system" ? undefined : timer.deliveryFrom,
|
|
230
|
+
component,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function normalizeSchedule(input: ScheduleTimerInput, now: number): { kind: TimerScheduleKind; def: TimerScheduleDef; nextRunAt: number } {
|
|
235
|
+
if (input.rrule || input.timezone) throw new ValidationError("rrule/timezone timers are not supported yet");
|
|
236
|
+
const modes = [input.afterMs !== undefined, input.at !== undefined, input.everyMs !== undefined].filter(Boolean).length;
|
|
237
|
+
if (modes !== 1) throw new ValidationError("provide exactly one of afterMs, at, or everyMs");
|
|
238
|
+
if (input.afterMs !== undefined) {
|
|
239
|
+
const afterMs = cleanPositiveInt(input.afterMs, "afterMs")!;
|
|
240
|
+
return { kind: "once", def: { afterMs }, nextRunAt: now + afterMs };
|
|
241
|
+
}
|
|
242
|
+
if (input.at !== undefined) {
|
|
243
|
+
const atMs = Date.parse(input.at);
|
|
244
|
+
if (!Number.isFinite(atMs) || atMs <= now) throw new ValidationError("at must be a future ISO timestamp");
|
|
245
|
+
return { kind: "once", def: { at: new Date(atMs).toISOString() }, nextRunAt: atMs };
|
|
246
|
+
}
|
|
247
|
+
const everyMs = cleanPositiveInt(input.everyMs, "everyMs")!;
|
|
248
|
+
return { kind: "interval", def: { everyMs }, nextRunAt: now + everyMs };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function normalizeTarget(target: string | undefined, ctx: AuthContext): string {
|
|
252
|
+
const raw = cleanOptionalString(target, "target", 200) ?? "me";
|
|
253
|
+
if (raw !== "me") return raw;
|
|
254
|
+
if (!ctx.callerAgentId) throw new ValidationError("target defaults to me, but caller has no single agent identity");
|
|
255
|
+
return ctx.callerAgentId;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function cleanBody(value: unknown): string {
|
|
259
|
+
if (typeof value !== "string" || !value.trim()) throw new ValidationError("body required");
|
|
260
|
+
const body = value.trim();
|
|
261
|
+
if (new TextEncoder().encode(body).byteLength > MAX_BODY_BYTES) throw new ValidationError(`body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
262
|
+
return body;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function cleanPayload(value: unknown): Record<string, unknown> | undefined {
|
|
266
|
+
if (value === undefined || value === null) return undefined;
|
|
267
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new ValidationError("payload must be an object");
|
|
268
|
+
const payload = value as Record<string, unknown>;
|
|
269
|
+
if (new TextEncoder().encode(JSON.stringify(payload)).byteLength > SCHEDULER_MAX_PAYLOAD_BYTES) {
|
|
270
|
+
throw new ValidationError(`payload exceeds ${SCHEDULER_MAX_PAYLOAD_BYTES} bytes`);
|
|
271
|
+
}
|
|
272
|
+
return payload;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function cleanOptionalString(value: unknown, field: string, max: number): string | undefined {
|
|
276
|
+
if (value === undefined || value === null) return undefined;
|
|
277
|
+
if (typeof value !== "string") throw new ValidationError(`${field} must be a string`);
|
|
278
|
+
const trimmed = value.trim();
|
|
279
|
+
if (!trimmed) return undefined;
|
|
280
|
+
if (trimmed.length > max) throw new ValidationError(`${field} must be ${max} characters or fewer`);
|
|
281
|
+
return trimmed;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function cleanPositiveInt(value: unknown, field: string): number | undefined {
|
|
285
|
+
if (value === undefined || value === null) return undefined;
|
|
286
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) throw new ValidationError(`${field} must be a positive integer`);
|
|
287
|
+
return value;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function cleanExpiresAt(value: unknown, now: number): number | undefined {
|
|
291
|
+
if (value === undefined || value === null) return undefined;
|
|
292
|
+
const ms = typeof value === "number" ? value : typeof value === "string" ? Date.parse(value) : NaN;
|
|
293
|
+
if (!Number.isFinite(ms) || ms <= now) throw new ValidationError("expiresAt must be a future ISO timestamp or unix timestamp in milliseconds");
|
|
294
|
+
return ms;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function canAdministerTimers(ctx: AuthContext): boolean {
|
|
298
|
+
return ctx.actor.kind === "system" || hasScope(ctx.scopes, "system:admin");
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function canCreatePersistentTimer(ctx: AuthContext): boolean {
|
|
302
|
+
return canAdministerTimers(ctx) || hasScope(ctx.scopes, "schedule:persistent");
|
|
303
|
+
}
|