agent-relay-server 0.54.0 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +1 -1
- package/src/bus.ts +3 -8
- package/src/db/agents.ts +8 -4
- package/src/db/claim-sql.ts +28 -0
- package/src/db/message-reads.ts +38 -4
- package/src/db/stats.ts +2 -1
- package/src/db/tasks.ts +2 -1
- package/src/maintenance.ts +3 -1
- package/src/provider-state.ts +66 -0
- package/src/services/rate-limit-resume.ts +96 -0
- package/src/services/send-message.ts +13 -1
- package/src/services/transient-agent-reaper.ts +5 -1
package/docs/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "Agent Relay API",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.56.0",
|
|
6
6
|
"description": "Real-time message bus for inter-agent communication. Agent-first: this spec is designed for machine consumption — agents can self-discover the full API surface via GET /api/spec.",
|
|
7
7
|
"license": {
|
|
8
8
|
"name": "MIT",
|
package/package.json
CHANGED
package/src/bus.ts
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
type RegisterFrame,
|
|
23
23
|
} from "agent-relay-sdk/protocol";
|
|
24
24
|
import { errMessage, isRecord, stringValue } from "agent-relay-sdk";
|
|
25
|
+
import { agentProviderState } from "./provider-state";
|
|
25
26
|
import { getComponentAuth, isComponentAuthorizedFor, isAuthorized, isOriginAllowed, unauthorized } from "./security";
|
|
26
27
|
import type { AgentCard, Command, ComponentToken, ContextState, Message, ProviderCapabilities, RegisterAgentInput, Task } from "./types";
|
|
27
28
|
|
|
@@ -532,8 +533,8 @@ function emitAgentStatusEvent(agentId: string): void {
|
|
|
532
533
|
}
|
|
533
534
|
|
|
534
535
|
function auditProviderStateTransition(agentId: string, before: AgentCard | null | undefined, after: AgentCard | null | undefined): void {
|
|
535
|
-
const previous =
|
|
536
|
-
const next =
|
|
536
|
+
const previous = agentProviderState(before);
|
|
537
|
+
const next = agentProviderState(after);
|
|
537
538
|
const previousKey = providerStateKey(previous);
|
|
538
539
|
const nextKey = providerStateKey(next);
|
|
539
540
|
if (previousKey === nextKey) return;
|
|
@@ -602,12 +603,6 @@ function auditRunnerTimelineEvent(agentId: string, timelineEvent: unknown): void
|
|
|
602
603
|
}
|
|
603
604
|
}
|
|
604
605
|
|
|
605
|
-
function providerStateFromAgent(agent: AgentCard | null | undefined): Record<string, unknown> | null {
|
|
606
|
-
const value = agent?.meta?.providerState;
|
|
607
|
-
if (!isRecord(value) || typeof value.state !== "string") return null;
|
|
608
|
-
return value;
|
|
609
|
-
}
|
|
610
|
-
|
|
611
606
|
function providerStateKey(state: Record<string, unknown> | null): string | null {
|
|
612
607
|
if (!state) return null;
|
|
613
608
|
return [state.state, typeof state.reason === "string" ? state.reason : ""].join(":");
|
package/src/db/agents.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { STALE_TTL_MS, DAY_MS, CLAIM_LEASE_MS, POOL_CLAIM_LEASE_MS, WORKSPACE_ME
|
|
|
16
16
|
import { matchAgents } from "../agent-ref";
|
|
17
17
|
import { evaluatePoolBindings, expectedChannelAgentId, upsertChannelForAgent } from "./channels.ts";
|
|
18
18
|
import { seedContinuationObjectiveFromSpawnPrompt } from "./continuations.ts";
|
|
19
|
+
import { messageTaskClosed } from "./claim-sql.ts";
|
|
19
20
|
import { ValidationError, getDb } from "./connection.ts";
|
|
20
21
|
import { rowToAgent, rowToContextSnapshot } from "./mappers.ts";
|
|
21
22
|
import { closeOpenPairsForAgent } from "./pairs.ts";
|
|
@@ -498,10 +499,12 @@ export function pruneOfflineAgents(maxOfflineMs: number = DAY_MS): string[] {
|
|
|
498
499
|
if (rows.length === 0) return [];
|
|
499
500
|
const now = Date.now();
|
|
500
501
|
|
|
501
|
-
// Release claims held by pruned agents so work becomes claimable again
|
|
502
|
+
// Release claims held by pruned agents so unfinished work becomes claimable again — but
|
|
503
|
+
// never for a message whose task is already closed: nulling that claim resurrects completed
|
|
504
|
+
// work on the next restart (the task release below is already guarded the same way).
|
|
502
505
|
getDb()
|
|
503
506
|
.query(
|
|
504
|
-
|
|
507
|
+
`UPDATE messages SET claimed_by = NULL, claimed_at = NULL, claim_expires_at = NULL WHERE claimed_by IN (SELECT id FROM agents WHERE status = 'offline' AND last_seen < ? AND id NOT IN ('user', 'system')) AND NOT ${messageTaskClosed("messages.id")}`
|
|
505
508
|
)
|
|
506
509
|
.run(cutoff);
|
|
507
510
|
|
|
@@ -534,10 +537,11 @@ export function deleteAgent(id: string): { ok: boolean; error?: string } {
|
|
|
534
537
|
return { ok: false, error: "built-in agent cannot be deleted" };
|
|
535
538
|
}
|
|
536
539
|
const deleted = getDb().transaction(() => {
|
|
537
|
-
// Release any claims held by this agent so the tasks become claimable again
|
|
540
|
+
// Release any claims held by this agent so the tasks become claimable again — except for
|
|
541
|
+
// messages whose task is already closed (releasing those resurrects finished work).
|
|
538
542
|
// from_agent is left intact as historical record.
|
|
539
543
|
const now = Date.now();
|
|
540
|
-
getDb().query(
|
|
544
|
+
getDb().query(`UPDATE messages SET claimed_by = NULL, claimed_at = NULL, claim_expires_at = NULL WHERE claimed_by = ? AND NOT ${messageTaskClosed("messages.id")}`).run(id);
|
|
541
545
|
settleSingleTargetOnDemandTasks("claimed_by = ?", [id], now, "agent-removed");
|
|
542
546
|
getDb().query("UPDATE tasks SET status = 'open', claimed_by = NULL, claimed_at = NULL, claim_expires_at = NULL, updated_at = ? WHERE claimed_by = ? AND status IN ('claimed', 'in_progress', 'blocked')").run(now, id);
|
|
543
547
|
revokeRuntimeTokensForAgent(id, now);
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Single home for the "is this message's task already finished?" SQL guard.
|
|
2
|
+
//
|
|
3
|
+
// A claimable message backed by a task must stop being treated as available work the moment
|
|
4
|
+
// the task is closed. This predicate had drifted: `releaseExpiredClaims` and the
|
|
5
|
+
// `expired-message-claims` health check carried it, but `pruneOfflineAgents`/`deleteAgent`
|
|
6
|
+
// (which release a pruned agent's message claims to NULL) and the poll's null-claim branch did
|
|
7
|
+
// NOT — so a completed automation briefing's message would get its claim nulled on agent prune
|
|
8
|
+
// and then re-deliver as fresh work to the next managed-agent restart (which always mints a new
|
|
9
|
+
// agent id, defeating the per-agent `message_reads` dedup). Route every site through here so the
|
|
10
|
+
// closed-task statuses live in exactly one place.
|
|
11
|
+
|
|
12
|
+
/** Task statuses that mean "finished — no more work to hand out for this message." */
|
|
13
|
+
const CLOSED_TASK_STATUSES = ["done", "failed", "canceled"] as const;
|
|
14
|
+
|
|
15
|
+
const CLOSED_TASK_STATUS_SQL = CLOSED_TASK_STATUSES.map((status) => `'${status}'`).join(", ");
|
|
16
|
+
|
|
17
|
+
/** SQL columns that reference a `messages.id` in the queries below. Internal, never user input. */
|
|
18
|
+
type MessageIdRef = "m.id" | "messages.id";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* SQL predicate: the message identified by `messageRef` has a linked task that is already closed.
|
|
22
|
+
* Negate it (`NOT ${messageTaskClosed(...)}`) to gate "available work" queries — claim release
|
|
23
|
+
* and inbox polling — so a finished task's claimable message is never released-to-NULL or
|
|
24
|
+
* re-polled as fresh work. `messageRef` is a fixed internal column reference, not user input.
|
|
25
|
+
*/
|
|
26
|
+
export function messageTaskClosed(messageRef: MessageIdRef): string {
|
|
27
|
+
return `EXISTS (SELECT 1 FROM tasks t WHERE t.message_id = ${messageRef} AND t.status IN (${CLOSED_TASK_STATUS_SQL}))`;
|
|
28
|
+
}
|
package/src/db/message-reads.ts
CHANGED
|
@@ -18,7 +18,8 @@ import { getAgent } from "./agents.ts";
|
|
|
18
18
|
import { sendMessageWithResult } from "./messages.ts";
|
|
19
19
|
import { deleteArtifactLinksForEntity } from "./artifacts.ts";
|
|
20
20
|
import { getDb, timedQuery } from "./connection.ts";
|
|
21
|
-
import { legacyChannelTargets } from "./delivery.ts";
|
|
21
|
+
import { legacyChannelTargets, isChannelAgentId } from "./delivery.ts";
|
|
22
|
+
import { messageTaskClosed } from "./claim-sql.ts";
|
|
22
23
|
import { MSG_SELECT, rowToMessage } from "./mappers.ts";
|
|
23
24
|
import type {
|
|
24
25
|
AgentCard,
|
|
@@ -203,9 +204,13 @@ export function pollMessages(query: PollQuery): Message[] {
|
|
|
203
204
|
}
|
|
204
205
|
conditions.push("delivery_status != 'queued' AND delivery_status NOT IN ('failed', 'dead')");
|
|
205
206
|
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
|
|
207
|
+
// A claimable message is available to this agent when it is non-claimable, currently held by
|
|
208
|
+
// THIS agent, or unclaimed/expired AND its task is not already closed. The closed-task guard
|
|
209
|
+
// must cover BOTH the null-claim and the expired-claim branch: when a managed agent is
|
|
210
|
+
// pruned/deleted its message claims are released to NULL (see pruneOfflineAgents/deleteAgent),
|
|
211
|
+
// so without it a finished task's message resurrects as fresh work on the next restart (which
|
|
212
|
+
// mints a new agent id, so message_reads no longer dedups it) — the automation re-delivery bug.
|
|
213
|
+
conditions.push(`(claimable = 0 OR claimed_by = ? OR ((claimed_by IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= ?) AND NOT ${messageTaskClosed("m.id")}))`);
|
|
209
214
|
params.push(query.for, Date.now());
|
|
210
215
|
|
|
211
216
|
if (query.sinceId !== undefined) {
|
|
@@ -320,6 +325,35 @@ export function listPendingReplyObligations(agentId: string, limit = 20): ReplyO
|
|
|
320
325
|
.map((message) => replyObligationFromMessage(message, agentId));
|
|
321
326
|
}
|
|
322
327
|
|
|
328
|
+
/**
|
|
329
|
+
* Channel-answer linking: a channel agent answers the human OUT-OF-BAND through its bridge — e.g.
|
|
330
|
+
* `agent-relay /message telegram '{"action":"reply",...}'` — which stores an outbound to the channel
|
|
331
|
+
* agent with NO reply_to / thread / conversation linkage (the bridge protocol body is opaque to the
|
|
332
|
+
* relay, so the inbound's channel + conversation never make it onto the answer). Without linkage the
|
|
333
|
+
* obligation never clears, and the Stop hook nags the agent into a redundant `/reply` that routes back
|
|
334
|
+
* through the channel as a second message to the human (the Telegram double-message friction).
|
|
335
|
+
*
|
|
336
|
+
* When the agent posts an unlinked message into a channel agent and has EXACTLY ONE open obligation on
|
|
337
|
+
* that channel AT THAT MOMENT, the conversation is unambiguous: this send is the answer. We snapshot
|
|
338
|
+
* that here, at send time, by returning the obligation id to link the outbound to — a decision a later
|
|
339
|
+
* sibling conversation can never retroactively confuse (the reason query-time coverage is unsafe). When
|
|
340
|
+
* two conversations are open the send is unattributable; we leave it unlinked for an explicit `/reply`,
|
|
341
|
+
* which carries the conversation precisely. Provider-agnostic: keys only on the channel agent target,
|
|
342
|
+
* never on the body.
|
|
343
|
+
*
|
|
344
|
+
* Returns the inbound message id this outbound answers, or null when nothing should be auto-linked.
|
|
345
|
+
*/
|
|
346
|
+
export function channelObligationAnsweredBy(
|
|
347
|
+
agentId: string,
|
|
348
|
+
channelAgentId: string,
|
|
349
|
+
): number | null {
|
|
350
|
+
if (!isChannelAgentId(channelAgentId)) return null;
|
|
351
|
+
const open = listPendingReplyObligations(agentId, 50).filter(
|
|
352
|
+
(obligation) => obligation.kind === "channel.event" && obligation.from === channelAgentId,
|
|
353
|
+
);
|
|
354
|
+
return open.length === 1 ? open[0]!.messageId : null;
|
|
355
|
+
}
|
|
356
|
+
|
|
323
357
|
/**
|
|
324
358
|
* #413 / vent #121 — seed a pending reply obligation for a freshly-registered spawned worker so
|
|
325
359
|
* it cannot strand its verdict. A spawned agent gets its task as the positional spawn prompt, NOT
|
package/src/db/stats.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { STALE_TTL_MS, DAY_MS, CLAIM_LEASE_MS, POOL_CLAIM_LEASE_MS, WORKSPACE_ME
|
|
|
16
16
|
import { matchAgents } from "../agent-ref";
|
|
17
17
|
import { listAgents } from "./agents.ts";
|
|
18
18
|
import { listChannels } from "./channels.ts";
|
|
19
|
+
import { messageTaskClosed } from "./claim-sql.ts";
|
|
19
20
|
import { getDb } from "./connection.ts";
|
|
20
21
|
import type {
|
|
21
22
|
AgentCard,
|
|
@@ -245,7 +246,7 @@ export function getHealth(now: number = Date.now()): HealthReport {
|
|
|
245
246
|
});
|
|
246
247
|
|
|
247
248
|
const expiredMessageClaims = (getDb()
|
|
248
|
-
.query(
|
|
249
|
+
.query(`SELECT COUNT(*) as c FROM messages WHERE claimed_by IS NOT NULL AND (claim_expires_at IS NULL OR claim_expires_at <= ?) AND NOT ${messageTaskClosed("messages.id")}`)
|
|
249
250
|
.get(now) as any).c as number;
|
|
250
251
|
checks.push({
|
|
251
252
|
name: "expired-message-claims",
|
package/src/db/tasks.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { STALE_TTL_MS, DAY_MS, CLAIM_LEASE_MS, POOL_CLAIM_LEASE_MS, WORKSPACE_MERGE_LEASE_MS } from "../config";
|
|
16
16
|
import { matchAgents } from "../agent-ref";
|
|
17
17
|
import { getAgent, settleSingleTargetOnDemandTasks, validateAgentSession } from "./agents.ts";
|
|
18
|
+
import { messageTaskClosed } from "./claim-sql.ts";
|
|
18
19
|
import { ClaimError, getDb } from "./connection.ts";
|
|
19
20
|
import { rowToTask, rowToTaskEvent } from "./mappers.ts";
|
|
20
21
|
import { claimMessageRow } from "./messages.ts";
|
|
@@ -169,7 +170,7 @@ export function recordTaskEvent(taskId: number, input: {
|
|
|
169
170
|
export function releaseExpiredClaims(now: number = Date.now()): { messageIds: number[]; tasks: Task[] } {
|
|
170
171
|
return getDb().transaction(() => {
|
|
171
172
|
settleSingleTargetOnDemandTasks("claimed_by IS NOT NULL AND (claim_expires_at IS NULL OR claim_expires_at <= ?)", [now], now, "claim-lease-expired");
|
|
172
|
-
const releasableMessageClaim =
|
|
173
|
+
const releasableMessageClaim = `claimed_by IS NOT NULL AND (claim_expires_at IS NULL OR claim_expires_at <= ?) AND NOT ${messageTaskClosed("messages.id")}`;
|
|
173
174
|
const messageRows = getDb()
|
|
174
175
|
.query(`SELECT id FROM messages WHERE ${releasableMessageClaim}`)
|
|
175
176
|
.all(now) as any[];
|
package/src/maintenance.ts
CHANGED
|
@@ -63,6 +63,7 @@ import { notifySystemMessage } from "./notify";
|
|
|
63
63
|
import { pruneExpiredTokenRecords } from "./token-db";
|
|
64
64
|
import { dissolveTeamById } from "./services/team";
|
|
65
65
|
import { reapTransientAgents, resetTransientTrackerForTests } from "./services/transient-agent-reaper";
|
|
66
|
+
import { resumeRateLimitedAgents, resetRateLimitResumeTrackerForTests } from "./services/rate-limit-resume";
|
|
66
67
|
import type { Command, MaintenanceJob, MaintenanceJobRun } from "./types";
|
|
67
68
|
|
|
68
69
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
@@ -135,7 +136,7 @@ const orphanReapEnabled = () => process.env.AGENT_RELAY_ORPHAN_REAP !== "0";
|
|
|
135
136
|
// orchestratorId + session identity -> when we first saw it orphaned (and last reaped).
|
|
136
137
|
const orphanTracker = new Map<string, { firstOrphanedAt: number; lastReapAt?: number }>();
|
|
137
138
|
|
|
138
|
-
export function resetOrphanTrackerForTests(): void { orphanTracker.clear(); resetTransientTrackerForTests(); }
|
|
139
|
+
export function resetOrphanTrackerForTests(): void { orphanTracker.clear(); resetTransientTrackerForTests(); resetRateLimitResumeTrackerForTests(); }
|
|
139
140
|
|
|
140
141
|
interface MaintenanceJobDefinition {
|
|
141
142
|
id: string;
|
|
@@ -300,6 +301,7 @@ const definitions: MaintenanceJobDefinition[] = [
|
|
|
300
301
|
handler: reapOrphanedSessions,
|
|
301
302
|
},
|
|
302
303
|
{ id: "transient-agent-reaper", title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: reapTransientAgents },
|
|
304
|
+
{ id: "rate-limit-resume", title: "Rate-limit resume", description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: resumeRateLimitedAgents },
|
|
303
305
|
{
|
|
304
306
|
id: "team-reap",
|
|
305
307
|
title: "Team lifecycle close-out",
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { isRecord } from "agent-relay-sdk";
|
|
2
|
+
|
|
3
|
+
// One server-side home for reading an agent's provider runtime state (the
|
|
4
|
+
// `meta.providerState` that both Claude and Codex ride for blocked/approval
|
|
5
|
+
// states — see runner setProviderStatus). Imported by the bus transition audit,
|
|
6
|
+
// the transient-agent reaper (don't reap a held agent), and the rate-limit
|
|
7
|
+
// resume sweep. The dashboard has its own client-side copy in display.ts.
|
|
8
|
+
|
|
9
|
+
export interface ProviderStateView {
|
|
10
|
+
state: string;
|
|
11
|
+
reason?: string;
|
|
12
|
+
label?: string;
|
|
13
|
+
/** Unix ms the provider's limit window resets (rate-limit holds). */
|
|
14
|
+
resetAt?: number;
|
|
15
|
+
/** Unix ms the hold was entered (used for the resume fallback when resetAt is absent). */
|
|
16
|
+
enteredAt?: number;
|
|
17
|
+
/** Raw provider error_type, kept for empirical observation of which type a real limit fires. */
|
|
18
|
+
errorType?: string;
|
|
19
|
+
[key: string]: unknown;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface HasMeta {
|
|
23
|
+
meta?: Record<string, unknown> | null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The agent's provider runtime state, or null when none is set / malformed. */
|
|
27
|
+
export function agentProviderState(agent: HasMeta | null | undefined): ProviderStateView | null {
|
|
28
|
+
const value = agent?.meta?.providerState;
|
|
29
|
+
if (!isRecord(value) || typeof value.state !== "string") return null;
|
|
30
|
+
return value as ProviderStateView;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** True when the agent is in any blocked provider state (rate-limit, approval, model-unavailable). */
|
|
34
|
+
export function isProviderBlocked(agent: HasMeta | null | undefined): boolean {
|
|
35
|
+
return agentProviderState(agent)?.state === "blocked";
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The `reason` value a rate-limit hold carries (matches the runner's control-server seam). */
|
|
39
|
+
export const RATE_LIMIT_BLOCK_REASON = "rate_limit";
|
|
40
|
+
|
|
41
|
+
/** True when the agent is specifically held on a provider usage/rate limit. */
|
|
42
|
+
export function isRateLimitHold(agent: HasMeta | null | undefined): boolean {
|
|
43
|
+
const state = agentProviderState(agent);
|
|
44
|
+
return state?.state === "blocked" && state.reason === RATE_LIMIT_BLOCK_REASON;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Fallback window when the provider gave no reset time: retry after this long
|
|
48
|
+
// rather than waiting the full (unknown) limit window. Re-entering the hold on a
|
|
49
|
+
// still-limited retry is cheap, and a subscription limit almost always carries a
|
|
50
|
+
// reset time. Read at call-time so operators/tests can tune it.
|
|
51
|
+
const rateLimitFallbackMs = (): number =>
|
|
52
|
+
Number(process.env.AGENT_RELAY_RATE_LIMIT_FALLBACK_MS) || 60 * 60 * 1000;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* When a rate-limit hold should be lifted. The provider's `resetAt` is
|
|
56
|
+
* authoritative; otherwise fall back to `enteredAt + fallback window`. Null only
|
|
57
|
+
* when neither is known (shouldn't happen — the runner always stamps enteredAt).
|
|
58
|
+
*/
|
|
59
|
+
export function rateLimitResumeAt(state: ProviderStateView | null): number | null {
|
|
60
|
+
if (!state) return null;
|
|
61
|
+
if (typeof state.resetAt === "number" && Number.isFinite(state.resetAt)) return state.resetAt;
|
|
62
|
+
if (typeof state.enteredAt === "number" && Number.isFinite(state.enteredAt)) {
|
|
63
|
+
return state.enteredAt + rateLimitFallbackMs();
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { createActivityEvent, listAgents } from "../db";
|
|
2
|
+
import { notifySystemMessage } from "../notify";
|
|
3
|
+
import { agentProviderState, isRateLimitHold, rateLimitResumeAt } from "../provider-state";
|
|
4
|
+
|
|
5
|
+
// #286 Phase 2: auto-resume agents held on a usage/rate-limit stall once their
|
|
6
|
+
// window resets. The runner reports the hold as `providerState: blocked` (reason
|
|
7
|
+
// rate_limit) with a resetAt; this sweep finds holds whose reset has passed and
|
|
8
|
+
// delivers a waking `continue` message. The agent's next turn clears the hold
|
|
9
|
+
// (the runner drops rateLimitHold on busy), so the blocked badge lifts on its own.
|
|
10
|
+
//
|
|
11
|
+
// Fan-out safe by construction: it resumes EVERY held agent past its reset, not
|
|
12
|
+
// one. Durable across restarts: resetAt lives in the agent's persisted meta, and
|
|
13
|
+
// the runner keeps re-asserting the hold while connected, so a relay restart just
|
|
14
|
+
// re-reads it. The cooldown tracker only debounces the brief window between
|
|
15
|
+
// sending the resume and the agent actually going busy.
|
|
16
|
+
|
|
17
|
+
const envMsOrDefault = (name: string, fallback: number): number => {
|
|
18
|
+
const v = Number(process.env[name]);
|
|
19
|
+
return Number.isFinite(v) && v >= 0 ? v : fallback;
|
|
20
|
+
};
|
|
21
|
+
// Don't re-send a resume to the same agent within this window — covers the gap
|
|
22
|
+
// between delivery and the agent flipping to busy (which clears the hold).
|
|
23
|
+
const resumeCooldownMs = () => envMsOrDefault("AGENT_RELAY_RATE_LIMIT_RESUME_COOLDOWN_MS", 5 * 60 * 1000);
|
|
24
|
+
|
|
25
|
+
const resumeTracker = new Map<string, number>();
|
|
26
|
+
|
|
27
|
+
export function resetRateLimitResumeTrackerForTests(): void {
|
|
28
|
+
resumeTracker.clear();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const RESUME_BODY =
|
|
32
|
+
"▶️ Your usage/rate limit has reset — resuming. Continue the task you were working on when the limit interrupted you. If you had already finished, no action is needed.";
|
|
33
|
+
|
|
34
|
+
export function resumeRateLimitedAgents(): Record<string, unknown> {
|
|
35
|
+
const now = Date.now();
|
|
36
|
+
const cooldown = resumeCooldownMs();
|
|
37
|
+
const seen = new Set<string>();
|
|
38
|
+
const resumed: string[] = [];
|
|
39
|
+
const waiting: string[] = [];
|
|
40
|
+
const cooling: string[] = [];
|
|
41
|
+
|
|
42
|
+
for (const agent of listAgents()) {
|
|
43
|
+
if (!isRateLimitHold(agent)) continue;
|
|
44
|
+
seen.add(agent.id);
|
|
45
|
+
|
|
46
|
+
const state = agentProviderState(agent);
|
|
47
|
+
const resumeAt = rateLimitResumeAt(state);
|
|
48
|
+
if (resumeAt !== null && now < resumeAt) { waiting.push(agent.id); continue; }
|
|
49
|
+
|
|
50
|
+
const lastResume = resumeTracker.get(agent.id);
|
|
51
|
+
if (lastResume !== undefined && now - lastResume < cooldown) { cooling.push(agent.id); continue; }
|
|
52
|
+
resumeTracker.set(agent.id, now);
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
notifySystemMessage(agent.id, {
|
|
56
|
+
subject: "Usage limit reset — resume",
|
|
57
|
+
body: RESUME_BODY,
|
|
58
|
+
payload: {
|
|
59
|
+
kind: "agent.rate_limit_resume",
|
|
60
|
+
agentId: agent.id,
|
|
61
|
+
resetAt: state?.resetAt,
|
|
62
|
+
errorType: state?.errorType,
|
|
63
|
+
},
|
|
64
|
+
replyExpected: false,
|
|
65
|
+
});
|
|
66
|
+
} catch {
|
|
67
|
+
// A stale/unreachable target must never break the sweep; it store-aheads
|
|
68
|
+
// and retries next pass once the cooldown lapses.
|
|
69
|
+
resumeTracker.delete(agent.id);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
resumed.push(agent.id);
|
|
74
|
+
createActivityEvent({
|
|
75
|
+
clientId: `rate-limit-resume-${agent.id}-${now}`,
|
|
76
|
+
kind: "state",
|
|
77
|
+
title: "Rate-limit hold resumed",
|
|
78
|
+
body: `Usage window reset — sent resume to ${agent.label ?? agent.id}`,
|
|
79
|
+
meta: agent.id,
|
|
80
|
+
icon: "ti-player-play",
|
|
81
|
+
view: "agents",
|
|
82
|
+
agentId: agent.id,
|
|
83
|
+
metadata: {
|
|
84
|
+
source: "server",
|
|
85
|
+
maintenanceJobId: "rate-limit-resume",
|
|
86
|
+
resetAt: state?.resetAt,
|
|
87
|
+
errorType: state?.errorType,
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Forget agents no longer held so a future hold resumes immediately.
|
|
93
|
+
for (const id of resumeTracker.keys()) if (!seen.has(id)) resumeTracker.delete(id);
|
|
94
|
+
|
|
95
|
+
return { held: seen.size, resumed, waiting, cooling, tracked: resumeTracker.size };
|
|
96
|
+
}
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
// OUT OF SCOPE (transport-edge, excluded from the parity facet): automatic memory injection
|
|
23
23
|
// (needs request-derived context) and per-transport audit (HTTP domain row vs MCP tool-call row).
|
|
24
24
|
import { isMechanicalMessageKind, isRecord, isReservedAgentId } from "agent-relay-sdk";
|
|
25
|
-
import { ValidationError, getMessage, listAgents, sendMessageWithResult } from "../db";
|
|
25
|
+
import { ValidationError, getMessage, listAgents, sendMessageWithResult, channelObligationAnsweredBy } from "../db";
|
|
26
26
|
import { planSend, type DeliveryReceipt } from "../agent-ref";
|
|
27
27
|
import { emitMessageQueued, emitNewMessage } from "../sse";
|
|
28
28
|
import { getLifecycleManager } from "../lifecycle-manager";
|
|
@@ -159,6 +159,18 @@ export function sendMessageService(
|
|
|
159
159
|
// Resolve BEFORE authorize so both run against the canonical id (kills the HTTP-authorizes-raw
|
|
160
160
|
// vs MCP-authorizes-resolved drift).
|
|
161
161
|
const receipt = resolveSendTarget(input, input.from);
|
|
162
|
+
// Channel-answer linking: an agent answering the human through a channel bridge posts a body-less
|
|
163
|
+
// message to the channel agent (e.g. `agent-relay /message telegram '{"action":"reply",...}'`) that
|
|
164
|
+
// carries no reply/conversation linkage, so its reply obligation never clears and the Stop hook nags
|
|
165
|
+
// it into a redundant `/reply` that leaks back to the human as a second message. When the agent has
|
|
166
|
+
// exactly ONE open obligation on that channel this send is unambiguously the answer — link it here,
|
|
167
|
+
// at send time, so the obligation clears with no extra message. (`channelObligationAnsweredBy` is a
|
|
168
|
+
// no-op unless `to` resolved to a channel agent with a single open obligation; deciding at send time
|
|
169
|
+
// — not query time — keeps a later sibling conversation from retroactively mis-attributing it.)
|
|
170
|
+
if (!input.replyTo && !isMechanicalMessageKind(input.kind)) {
|
|
171
|
+
const answered = channelObligationAnsweredBy(input.from, input.to);
|
|
172
|
+
if (answered !== null) input.replyTo = answered;
|
|
173
|
+
}
|
|
162
174
|
authorizeSend(input, ctx, opts, reservedSink);
|
|
163
175
|
|
|
164
176
|
const result = sendMessageWithResult(input);
|
|
@@ -2,6 +2,7 @@ import { createCommand } from "../commands-db";
|
|
|
2
2
|
import { emitRelayEvent } from "../events";
|
|
3
3
|
import { createActivityEvent, getAgent, listAgents, listWorkspaces } from "../db";
|
|
4
4
|
import { notifySystemMessage } from "../notify";
|
|
5
|
+
import { isProviderBlocked } from "../provider-state";
|
|
5
6
|
import { TERMINAL_WORKSPACE_STATUSES } from "../workspace-phase";
|
|
6
7
|
|
|
7
8
|
const envMsOrDefault = (name: string, fallback: number): number => {
|
|
@@ -64,7 +65,10 @@ export function reapTransientAgents(): Record<string, unknown> {
|
|
|
64
65
|
for (const agent of listAgents()) {
|
|
65
66
|
if (agent.lifecycle !== "transient") continue;
|
|
66
67
|
seen.add(agent.id);
|
|
67
|
-
|
|
68
|
+
// A held agent (rate-limit hold #286, or any provider-blocked state) reads as
|
|
69
|
+
// idle+ready but is NOT done — reaping it would kill work that's just waiting
|
|
70
|
+
// for the limit to reset. Treat it as not-idle; the clock restarts on resume.
|
|
71
|
+
if (agent.status !== "idle" || !agent.ready || isProviderBlocked(agent)) { transientTracker.delete(agent.id); continue; }
|
|
68
72
|
const entry = transientTracker.get(agent.id) ?? { firstIdleAt: now };
|
|
69
73
|
transientTracker.set(agent.id, entry);
|
|
70
74
|
const idleForMs = now - entry.firstIdleAt;
|