@sema-agent/core 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/send-message-tool.js +37 -29
- package/dist/agents/subagent.js +91 -4
- package/dist/brain/circuit-breaker.js +18 -8
- package/dist/brain/retry.d.ts +1 -0
- package/dist/brain/retry.js +29 -7
- package/dist/brain/stream-engine.d.ts +1 -0
- package/dist/brain/stream-engine.js +74 -12
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +1 -0
- package/dist/core/auto-compaction.js +9 -1
- package/dist/core/background-agent-store.d.ts +2 -0
- package/dist/core/background-agent-store.js +20 -0
- package/dist/core/mcp.js +8 -5
- package/dist/core/runner/assemble-result.d.ts +1 -0
- package/dist/core/runner/assemble-result.js +1 -1
- package/dist/core/runner/prepare-task.d.ts +2 -1
- package/dist/core/runner/prepare-task.js +61 -20
- package/dist/core/runner/runtask.js +38 -12
- package/dist/core/runner/tool-disclosure.d.ts +8 -3
- package/dist/core/runner/tool-disclosure.js +39 -10
- package/dist/core/skills-directory.d.ts +1 -1
- package/dist/core/skills-directory.js +257 -28
- package/dist/core/task-registry-agent.d.ts +2 -1
- package/dist/core/task-registry-agent.js +50 -54
- package/dist/core/task-registry.d.ts +1 -0
- package/dist/core/task-registry.js +1 -1
- package/dist/core/types.d.ts +9 -1
- package/dist/engine/compaction/compaction.js +71 -20
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/internal/harness-types.d.ts +1 -1
- package/dist/internal/harness.d.ts +1 -1
- package/dist/internal/harness.js +1 -1
- package/dist/orchestration/run-workflow-tool.d.ts +1 -0
- package/dist/orchestration/run-workflow-tool.js +4 -1
- package/dist/orchestration/workflow.d.ts +1 -1
- package/dist/orchestration/workflow.js +20 -12
- package/dist/tools/fs/bash-readonly-classifier.js +83 -17
- package/dist/tools/fs/fs-bash.js +17 -11
- package/dist/tools/fs/fs-shared.d.ts +1 -0
- package/dist/tools/fs/fs-shared.js +44 -2
- package/dist/tools/web.d.ts +15 -0
- package/dist/tools/web.js +42 -0
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { defineTool } from "../core/tools.js";
|
|
3
3
|
import { normalizeAgentName, DURABLE_AGENT_HANDLE_RE, DURABLE_AGENT_HEARTBEAT_MS } from "../core/task-registry.js";
|
|
4
|
-
import { canAccessAgentRecord } from "../core/background-agent-store.js";
|
|
4
|
+
import { canAccessAgentRecord, clearRevivedRowTerminalPayload } from "../core/background-agent-store.js";
|
|
5
5
|
import { escapeAttributeValue, escapeEnvelopeTag, isObserverTaskId, OBSERVER_SENDMESSAGE_SENDER_REFUSAL, OBSERVER_SENDMESSAGE_TARGET_REFUSAL, } from "./observer.js";
|
|
6
6
|
import { SUBAGENT_RESUME_CAP, SubagentRetainLedger, getSessionRetainLedger } from "./retain-ledger.js";
|
|
7
7
|
import { createSubagentResume } from "./subagent.js";
|
|
@@ -91,8 +91,9 @@ export function createSendMessageTool(opts) {
|
|
|
91
91
|
}
|
|
92
92
|
const senderId = ctx.taskId ?? opts.owner;
|
|
93
93
|
const [parentTaskId, parentSessionId] = ctx.parentTaskId !== undefined ? [ctx.parentTaskId, ctx.parentSessionId] : [opts.parentTaskId, opts.parentSessionId];
|
|
94
|
-
const
|
|
95
|
-
const
|
|
94
|
+
const senderName = ctx.spawnedAgentName ?? opts.senderName;
|
|
95
|
+
const senderIsChild = parentTaskId !== undefined || opts.uplink !== undefined || senderName !== undefined || opts.siblingRetain !== undefined;
|
|
96
|
+
const senderLabel = senderName ?? (senderIsChild ? senderId ?? "main" : "main");
|
|
96
97
|
if (senderId !== undefined && isObserverTaskId(senderId)) {
|
|
97
98
|
return { content: OBSERVER_SENDMESSAGE_SENDER_REFUSAL, details: { error: "observer_sender" }, isError: true };
|
|
98
99
|
}
|
|
@@ -102,7 +103,7 @@ export function createSendMessageTool(opts) {
|
|
|
102
103
|
if (normalizeAgentName(to) === "main") {
|
|
103
104
|
if (opts.uplink && senderId !== undefined) {
|
|
104
105
|
const uplinkSummaryRaw = typeof a.summary === "string" && a.summary.trim() !== "" ? clipSendMessageSummary(a.summary.trim()) : message.slice(0, 80);
|
|
105
|
-
const fromLabel =
|
|
106
|
+
const fromLabel = senderName ?? senderId;
|
|
106
107
|
try {
|
|
107
108
|
opts.uplink({
|
|
108
109
|
task_id: senderId,
|
|
@@ -140,6 +141,9 @@ export function createSendMessageTool(opts) {
|
|
|
140
141
|
scope: ctx.principal ?? opts.scope,
|
|
141
142
|
...((ctx.sessionId ?? opts.sessionId) !== undefined ? { sessionId: ctx.sessionId ?? opts.sessionId } : {}),
|
|
142
143
|
};
|
|
144
|
+
const parentAccess = parentTaskId !== undefined
|
|
145
|
+
? { ...access, owner: parentTaskId, ...(parentSessionId !== undefined ? { sessionId: parentSessionId } : {}) }
|
|
146
|
+
: undefined;
|
|
143
147
|
const tier3Revive = async (handle) => {
|
|
144
148
|
if (opts.agentStore === undefined || opts.mailbox === undefined || opts.reviveSpawn === undefined)
|
|
145
149
|
return undefined;
|
|
@@ -156,10 +160,9 @@ export function createSendMessageTool(opts) {
|
|
|
156
160
|
if (!row)
|
|
157
161
|
return undefined;
|
|
158
162
|
if (!canAccessAgentRecord(row, access)) {
|
|
159
|
-
if (
|
|
163
|
+
if (parentAccess === undefined)
|
|
160
164
|
return undefined;
|
|
161
|
-
|
|
162
|
-
if (!canAccessAgentRecord(row, parentView))
|
|
165
|
+
if (!canAccessAgentRecord(row, parentAccess))
|
|
163
166
|
return undefined;
|
|
164
167
|
}
|
|
165
168
|
if (row.name === undefined || row.agentType === "fork")
|
|
@@ -213,15 +216,7 @@ export function createSendMessageTool(opts) {
|
|
|
213
216
|
const now = Date.now();
|
|
214
217
|
const nextSeq = (row.seq ?? 1) + 1;
|
|
215
218
|
const claimed = { ...row, status: "running", writerId: opts.registry.writerId, writerEpoch: (row.writerEpoch ?? 0) + 1, updatedAt: now, seq: nextSeq };
|
|
216
|
-
|
|
217
|
-
delete claimed.stoppedBy;
|
|
218
|
-
delete claimed.finalOutput;
|
|
219
|
-
delete claimed.error;
|
|
220
|
-
delete claimed.resultIsPartial;
|
|
221
|
-
delete claimed.summary;
|
|
222
|
-
delete claimed.recentSteps;
|
|
223
|
-
delete claimed.editedFiles;
|
|
224
|
-
delete claimed.usage;
|
|
219
|
+
clearRevivedRowTerminalPayload(claimed);
|
|
225
220
|
let won = false;
|
|
226
221
|
try {
|
|
227
222
|
won = await opts.agentStore.update(handle, scope, claimed, { rev: row.rev });
|
|
@@ -345,12 +340,11 @@ export function createSendMessageTool(opts) {
|
|
|
345
340
|
};
|
|
346
341
|
let idRow = opts.registry.getAccessibleTask(to, access);
|
|
347
342
|
let resolvedAccess = access;
|
|
348
|
-
if (!idRow &&
|
|
349
|
-
const
|
|
350
|
-
const siblingById = opts.registry.getAccessibleTask(to, parentView);
|
|
343
|
+
if (!idRow && parentAccess !== undefined) {
|
|
344
|
+
const siblingById = opts.registry.getAccessibleTask(to, parentAccess);
|
|
351
345
|
if (siblingById) {
|
|
352
346
|
idRow = siblingById;
|
|
353
|
-
resolvedAccess =
|
|
347
|
+
resolvedAccess = parentAccess;
|
|
354
348
|
}
|
|
355
349
|
}
|
|
356
350
|
if (!idRow && DURABLE_AGENT_HANDLE_RE.test(to)) {
|
|
@@ -361,12 +355,13 @@ export function createSendMessageTool(opts) {
|
|
|
361
355
|
let target = idRow;
|
|
362
356
|
if (!target) {
|
|
363
357
|
let byName = opts.registry.resolveBackgroundAgentByName(to, access);
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
const sibling = opts.registry.resolveBackgroundAgentByName(to,
|
|
358
|
+
let parentByName;
|
|
359
|
+
if (byName.status === "not_found" && parentAccess !== undefined) {
|
|
360
|
+
const sibling = opts.registry.resolveBackgroundAgentByName(to, parentAccess);
|
|
361
|
+
parentByName = sibling;
|
|
367
362
|
if (sibling.status === "found" || sibling.status === "ambiguous") {
|
|
368
363
|
byName = sibling;
|
|
369
|
-
resolvedAccess =
|
|
364
|
+
resolvedAccess = parentAccess;
|
|
370
365
|
}
|
|
371
366
|
}
|
|
372
367
|
if (byName.status === "ambiguous") {
|
|
@@ -381,6 +376,8 @@ export function createSendMessageTool(opts) {
|
|
|
381
376
|
if (opts.roster) {
|
|
382
377
|
try {
|
|
383
378
|
rosterHit = await opts.roster.resolve(to, access);
|
|
379
|
+
if (rosterHit === undefined && parentAccess !== undefined)
|
|
380
|
+
rosterHit = await opts.roster.resolve(to, parentAccess);
|
|
384
381
|
}
|
|
385
382
|
catch {
|
|
386
383
|
}
|
|
@@ -396,13 +393,17 @@ export function createSendMessageTool(opts) {
|
|
|
396
393
|
isError: true,
|
|
397
394
|
};
|
|
398
395
|
}
|
|
399
|
-
const
|
|
396
|
+
const suggestion = byName.suggestion ?? (parentByName?.status === "not_found" ? parentByName.suggestion : undefined);
|
|
397
|
+
const ownLabels = opts.registry.runningBackgroundAgentLabels(access);
|
|
398
|
+
const labels = parentAccess !== undefined
|
|
399
|
+
? [...ownLabels, ...opts.registry.runningBackgroundAgentLabels(parentAccess).filter((l) => !ownLabels.includes(l))]
|
|
400
|
+
: ownLabels;
|
|
400
401
|
return {
|
|
401
402
|
content: `Message not sent: no agent matches "${to}" (unknown id or name, not yours, or expired).` +
|
|
402
|
-
(
|
|
403
|
+
(suggestion !== undefined ? ` Did you mean: ${suggestion}?` : "") +
|
|
403
404
|
(labels.length > 0 ? ` Running background agents: ${labels.join(", ")}.` : "") +
|
|
404
405
|
` Note: a completed foreground agent is not resumable (its transcript is not retained) — spawn with run_in_background to keep an agent addressable, or launch a new agent.`,
|
|
405
|
-
details: { error: "not_found", to, ...(
|
|
406
|
+
details: { error: "not_found", to, ...(suggestion !== undefined ? { suggestion } : {}) },
|
|
406
407
|
isError: true,
|
|
407
408
|
};
|
|
408
409
|
}
|
|
@@ -424,7 +425,7 @@ export function createSendMessageTool(opts) {
|
|
|
424
425
|
: message;
|
|
425
426
|
const teammateXml = frameTeammateMessage({ from: fromLabel, ...(s2Summary !== undefined ? { summary: s2Summary } : {}), text: s2Clipped });
|
|
426
427
|
const delivered = await opts.registry.deliverToRunningAgent(targetId, resolvedAccess, {
|
|
427
|
-
task_id:
|
|
428
|
+
task_id: senderLabel,
|
|
428
429
|
task_type: "background_agent",
|
|
429
430
|
status: "event",
|
|
430
431
|
summary: `message from ${fromLabel}${s2Summary !== undefined ? `: ${s2Summary}` : ""}`,
|
|
@@ -450,6 +451,13 @@ export function createSendMessageTool(opts) {
|
|
|
450
451
|
};
|
|
451
452
|
}
|
|
452
453
|
const nowRow = opts.registry.getAccessibleTask(targetId, resolvedAccess);
|
|
454
|
+
if (nowRow?.status === "parked") {
|
|
455
|
+
return {
|
|
456
|
+
content: `Message not sent: ${who} is parked on a pending approval — it resumes when the approval is decided, not by message delivery. Send again after it resumes.`,
|
|
457
|
+
details: { error: "parked_pending_approval", to },
|
|
458
|
+
isError: true,
|
|
459
|
+
};
|
|
460
|
+
}
|
|
453
461
|
const stillLive = nowRow !== undefined && (nowRow.status === "running" || nowRow.status === "pending");
|
|
454
462
|
return stillLive
|
|
455
463
|
? {
|
|
@@ -524,7 +532,7 @@ export function createSendMessageTool(opts) {
|
|
|
524
532
|
...(opts.notify ? { currentParentNotify: opts.notify } : {}),
|
|
525
533
|
});
|
|
526
534
|
const summary = typeof a.summary === "string" && a.summary.trim() !== "" ? clipSendMessageSummary(a.summary.trim()) : undefined;
|
|
527
|
-
const fromPrefix = parentTaskId !== undefined ? `(message from teammate "${
|
|
535
|
+
const fromPrefix = parentTaskId !== undefined ? `(message from teammate "${senderName ?? senderId ?? "unknown"}")\n` : "";
|
|
528
536
|
try {
|
|
529
537
|
const safeSummary = summary !== undefined ? escapeEnvelopeTag(TEAMMATE_MESSAGE_TAG, summary) : undefined;
|
|
530
538
|
const safeMessage = escapeEnvelopeTag(TEAMMATE_MESSAGE_TAG, message);
|
package/dist/agents/subagent.js
CHANGED
|
@@ -17,7 +17,7 @@ import { addWorktree } from "../core/git-worktree-env.js";
|
|
|
17
17
|
import { shellQuote } from "../tools/fs/search.js";
|
|
18
18
|
import { BG_AGENT_REAP_STOP_ERROR } from "../core/task-registry.js";
|
|
19
19
|
import { extractErrorCode } from "../brain/errors.js";
|
|
20
|
-
import { FORK_DEFAULT_MAX_TURNS, SESSION_BG_DEFAULT_TIMEOUT_SEC, RETAIN_DEFAULT_TTL_MS, RETAIN_DEFAULT_MAX } from "../config/defaults.js";
|
|
20
|
+
import { FORK_DEFAULT_MAX_TURNS, SESSION_BG_DEFAULT_TIMEOUT_SEC, RETAIN_DEFAULT_TTL_MS, RETAIN_DEFAULT_MAX, RUNNING_AGENT_OBSERVE_EVERY_BEATS } from "../config/defaults.js";
|
|
21
21
|
export { FORK_DEFAULT_MAX_TURNS, SESSION_BG_DEFAULT_TIMEOUT_SEC, RETAIN_DEFAULT_TTL_MS, RETAIN_DEFAULT_MAX };
|
|
22
22
|
import { SUBAGENT_RESUME_CAP, SubagentRetainLedger, getOrCreateSessionRetainLedger, ensureSessionReapHook, createResumePrompt, } from "./retain-ledger.js";
|
|
23
23
|
import { recordRosterSpawn } from "./roster-store.js";
|
|
@@ -227,6 +227,21 @@ function errorKindClause(c) {
|
|
|
227
227
|
const FAILED_SESSION_RETAIN_TTL_MS = 15 * 60 * 1000;
|
|
228
228
|
const PARTIAL_FINDINGS_MAX_CHARS = 1200;
|
|
229
229
|
const BG_NOTIFY_DRAIN_WINDOW_MS = 2_000;
|
|
230
|
+
function createBgActivityBeat(parentToolCallId, emitTick) {
|
|
231
|
+
let beats = 0;
|
|
232
|
+
let starts = 0;
|
|
233
|
+
return (e) => {
|
|
234
|
+
if (e.type !== "tool_start" && e.type !== "tool_end")
|
|
235
|
+
return;
|
|
236
|
+
if (parentToolCallId === undefined || e.parentToolCallId !== parentToolCallId)
|
|
237
|
+
return;
|
|
238
|
+
if (e.type === "tool_start")
|
|
239
|
+
starts += 1;
|
|
240
|
+
beats += 1;
|
|
241
|
+
if (beats === 1 || beats % RUNNING_AGENT_OBSERVE_EVERY_BEATS === 0)
|
|
242
|
+
emitTick(starts);
|
|
243
|
+
};
|
|
244
|
+
}
|
|
230
245
|
function markerFragment() {
|
|
231
246
|
return uuidv7().replace(/-/g, "").slice(-12);
|
|
232
247
|
}
|
|
@@ -351,6 +366,25 @@ export function createSubagentResume(deps) {
|
|
|
351
366
|
}
|
|
352
367
|
}
|
|
353
368
|
reviveStartedAt = Date.now();
|
|
369
|
+
const reviveActivityBeat = createBgActivityBeat(deps.parentToolCallId, (toolStarts) => {
|
|
370
|
+
if (reviveEmit === undefined)
|
|
371
|
+
return;
|
|
372
|
+
const currentAction = resumeStepRecorder.currentAction();
|
|
373
|
+
const currentTool = resumeStepRecorder.currentActionStructured();
|
|
374
|
+
reviveEmit({
|
|
375
|
+
kind: "tick",
|
|
376
|
+
taskId: deps.taskId,
|
|
377
|
+
sessionScoped: deps.sessionScoped === true,
|
|
378
|
+
...(deps.rowAgentType !== undefined ? { agentType: deps.rowAgentType } : {}),
|
|
379
|
+
transcriptId: entry.childSessionId,
|
|
380
|
+
sessionId: entry.childSessionId,
|
|
381
|
+
...(deps.parentToolCallId !== undefined ? { parentToolCallId: deps.parentToolCallId } : {}),
|
|
382
|
+
progressTaskId: entry.childSessionId,
|
|
383
|
+
...(currentAction !== undefined ? { currentAction } : {}),
|
|
384
|
+
...(currentTool !== undefined ? { currentTool } : {}),
|
|
385
|
+
usage: { toolUses: toolStarts },
|
|
386
|
+
});
|
|
387
|
+
});
|
|
354
388
|
stream = childRunner.runTaskStream(resumeSpec, undefined, {
|
|
355
389
|
...entry.internalsSnapshot,
|
|
356
390
|
...(true
|
|
@@ -362,6 +396,7 @@ export function createSubagentResume(deps) {
|
|
|
362
396
|
}
|
|
363
397
|
catch {
|
|
364
398
|
}
|
|
399
|
+
reviveActivityBeat(e);
|
|
365
400
|
if (reviveEmit !== undefined && e.type === "task_progress") {
|
|
366
401
|
reviveEmit({
|
|
367
402
|
kind: "tick",
|
|
@@ -1201,6 +1236,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1201
1236
|
tools: [reportTool],
|
|
1202
1237
|
...(ctx.excludeTools !== undefined ? { excludeTools: [...ctx.excludeTools] } : {}),
|
|
1203
1238
|
...(ctx.deferTools !== undefined ? { deferTools: [...ctx.deferTools] } : {}),
|
|
1239
|
+
...(ctx.alwaysLoadTools !== undefined ? { alwaysLoadTools: [...ctx.alwaysLoadTools] } : {}),
|
|
1204
1240
|
...(ctx.promptProfile !== undefined ? { promptProfile: ctx.promptProfile } : {}),
|
|
1205
1241
|
enableBlockedReport: false,
|
|
1206
1242
|
limits: { maxTurns: observerDef.maxTurns ?? 8, timeoutSec: 120 },
|
|
@@ -1344,9 +1380,17 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1344
1380
|
parentToolCallId: ctx.toolCallId,
|
|
1345
1381
|
...(childAgentName ? { agentName: childAgentName } : {}),
|
|
1346
1382
|
...(agentName !== undefined ? { explicitAgentName: agentName } : {}),
|
|
1347
|
-
...(
|
|
1348
|
-
|
|
1349
|
-
|
|
1383
|
+
...(reviveClaim !== undefined
|
|
1384
|
+
? {
|
|
1385
|
+
...(reviveClaim.row.parentTaskId !== undefined ? { parentTaskId: reviveClaim.row.parentTaskId } : {}),
|
|
1386
|
+
...(reviveClaim.row.parentSessionId !== undefined ? { parentSessionId: reviveClaim.row.parentSessionId } : {}),
|
|
1387
|
+
...(reviveClaim.row.rootSessionId !== undefined ? { rootSessionId: reviveClaim.row.rootSessionId } : {}),
|
|
1388
|
+
}
|
|
1389
|
+
: {
|
|
1390
|
+
...(ctx.taskId !== undefined ? { parentTaskId: ctx.taskId } : {}),
|
|
1391
|
+
...(ctx.sessionId !== undefined ? { parentSessionId: ctx.sessionId } : {}),
|
|
1392
|
+
...((ctx.rootSessionId ?? ctx.sessionId) !== undefined ? { rootSessionId: ctx.rootSessionId ?? ctx.sessionId } : {}),
|
|
1393
|
+
}),
|
|
1350
1394
|
...(ctx.centerArtifactDigest !== undefined ? { parentCenterArtifactDigest: ctx.centerArtifactDigest } : {}),
|
|
1351
1395
|
...(ctx.centerSourceRevision !== undefined ? { parentCenterSourceRevision: ctx.centerSourceRevision } : {}),
|
|
1352
1396
|
...(ctx.onTaskNotification !== undefined ? { parentNotify: ctx.onTaskNotification } : {}),
|
|
@@ -1375,6 +1419,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1375
1419
|
...(ctx.clientContext !== undefined ? { clientContext: ctx.clientContext } : {}),
|
|
1376
1420
|
...(ctx.excludeTools !== undefined ? { excludeTools: [...ctx.excludeTools] } : {}),
|
|
1377
1421
|
...(ctx.deferTools !== undefined ? { deferTools: [...ctx.deferTools] } : {}),
|
|
1422
|
+
...(ctx.alwaysLoadTools !== undefined ? { alwaysLoadTools: [...ctx.alwaysLoadTools] } : {}),
|
|
1378
1423
|
...(ctx.promptProfile !== undefined ? { promptProfile: ctx.promptProfile } : {}),
|
|
1379
1424
|
...(ctx.additionalDirectories !== undefined ? { additionalDirectories: [...ctx.additionalDirectories] } : {}),
|
|
1380
1425
|
...(ctx.envFacts !== undefined ? { envFacts: { ...ctx.envFacts } } : {}),
|
|
@@ -1402,6 +1447,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1402
1447
|
...(plainSpec.clientContext !== undefined ? { clientContext: { ...plainSpec.clientContext } } : {}),
|
|
1403
1448
|
...(plainSpec.excludeTools !== undefined ? { excludeTools: [...plainSpec.excludeTools] } : {}),
|
|
1404
1449
|
...(plainSpec.deferTools !== undefined ? { deferTools: [...plainSpec.deferTools] } : {}),
|
|
1450
|
+
...(plainSpec.alwaysLoadTools !== undefined ? { alwaysLoadTools: [...plainSpec.alwaysLoadTools] } : {}),
|
|
1405
1451
|
}),
|
|
1406
1452
|
internalsSnapshot: { ...childInternals },
|
|
1407
1453
|
release: async () => {
|
|
@@ -1482,6 +1528,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1482
1528
|
...(plainSpec.clientContext !== undefined ? { clientContext: { ...plainSpec.clientContext } } : {}),
|
|
1483
1529
|
...(plainSpec.excludeTools !== undefined ? { excludeTools: [...plainSpec.excludeTools] } : {}),
|
|
1484
1530
|
...(plainSpec.deferTools !== undefined ? { deferTools: [...plainSpec.deferTools] } : {}),
|
|
1531
|
+
...(plainSpec.alwaysLoadTools !== undefined ? { alwaysLoadTools: [...plainSpec.alwaysLoadTools] } : {}),
|
|
1485
1532
|
}),
|
|
1486
1533
|
internalsSnapshot: { ...childInternals },
|
|
1487
1534
|
release: releaseChild,
|
|
@@ -1686,6 +1733,23 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1686
1733
|
const s2ForkNotifyReady = (inject) => {
|
|
1687
1734
|
bg.registry.attachAgentNotify(taskId, inject);
|
|
1688
1735
|
};
|
|
1736
|
+
const bgForkActivityBeat = createBgActivityBeat(ctx.toolCallId, (toolStarts) => {
|
|
1737
|
+
const currentAction = stepRecorder.currentAction();
|
|
1738
|
+
const currentTool = stepRecorder.currentActionStructured();
|
|
1739
|
+
sinkEmit({
|
|
1740
|
+
kind: "tick",
|
|
1741
|
+
taskId,
|
|
1742
|
+
sessionScoped: sessionScopedBg === true,
|
|
1743
|
+
transcriptId: forkedId,
|
|
1744
|
+
sessionId: forkedId,
|
|
1745
|
+
parentToolCallId: ctx.toolCallId,
|
|
1746
|
+
progressTaskId: forkedId,
|
|
1747
|
+
agentType: spawnAgentType,
|
|
1748
|
+
...(currentAction !== undefined ? { currentAction } : {}),
|
|
1749
|
+
...(currentTool !== undefined ? { currentTool } : {}),
|
|
1750
|
+
usage: { toolUses: toolStarts },
|
|
1751
|
+
});
|
|
1752
|
+
});
|
|
1689
1753
|
const bgForkInternals = bgSink
|
|
1690
1754
|
? {
|
|
1691
1755
|
...forkInternals,
|
|
@@ -1696,6 +1760,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1696
1760
|
}
|
|
1697
1761
|
catch {
|
|
1698
1762
|
}
|
|
1763
|
+
bgForkActivityBeat(e);
|
|
1699
1764
|
if (e.type === "task_progress") {
|
|
1700
1765
|
const currentAction = stepRecorder.currentAction();
|
|
1701
1766
|
const currentTool = stepRecorder.currentActionStructured();
|
|
@@ -1752,6 +1817,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1752
1817
|
? classifySubagentError({ status: "failed", ...(child.errorCode !== undefined ? { errorCode: child.errorCode } : {}), ...(child.errorMessage !== undefined ? { errorMessage: child.errorMessage } : {}) })
|
|
1753
1818
|
: undefined;
|
|
1754
1819
|
const settledBg = bg.registry.settleBackgroundAgent(taskId, {
|
|
1820
|
+
cycle: 0,
|
|
1755
1821
|
status: okBg ? "completed" : reapedBg ? "killed" : "failed",
|
|
1756
1822
|
...resultSettleFields(child.result),
|
|
1757
1823
|
...(!okBg ? { error: reapedBg ? (collateralBg ? BG_AGENT_COLLATERAL_REAP_REASON : BG_AGENT_REAP_STOP_ERROR) : child.errorMessage ?? String(child.status) } : {}),
|
|
@@ -1823,6 +1889,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
1823
1889
|
const errCodeForkReject = killed ? undefined : extractErrorCode(msgFork);
|
|
1824
1890
|
const errClassForkReject = killed ? undefined : classifySubagentError({ status: "failed", errorMessage: msgFork });
|
|
1825
1891
|
const settledBg = bg.registry.settleBackgroundAgent(taskId, {
|
|
1892
|
+
cycle: 0,
|
|
1826
1893
|
status: killed ? "killed" : "failed",
|
|
1827
1894
|
error: killed ? BG_AGENT_REAP_STOP_ERROR : msgFork,
|
|
1828
1895
|
...(errCodeForkReject !== undefined ? { errorCode: errCodeForkReject } : {}),
|
|
@@ -2145,6 +2212,23 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
2145
2212
|
bg.registry.finalizeParkedResume(taskId);
|
|
2146
2213
|
reviveAttachedResolve?.();
|
|
2147
2214
|
};
|
|
2215
|
+
const bgActivityBeat = createBgActivityBeat(ctx.toolCallId, (toolStarts) => {
|
|
2216
|
+
const currentAction = stepRecorder.currentAction();
|
|
2217
|
+
const currentTool = stepRecorder.currentActionStructured();
|
|
2218
|
+
sinkEmit({
|
|
2219
|
+
kind: "tick",
|
|
2220
|
+
taskId,
|
|
2221
|
+
sessionScoped: sessionScopedBg === true,
|
|
2222
|
+
transcriptId: bgChildSessionId,
|
|
2223
|
+
sessionId: bgChildSessionId,
|
|
2224
|
+
parentToolCallId: ctx.toolCallId,
|
|
2225
|
+
progressTaskId: bgChildSessionId,
|
|
2226
|
+
agentType: spawnAgentType,
|
|
2227
|
+
...(currentAction !== undefined ? { currentAction } : {}),
|
|
2228
|
+
...(currentTool !== undefined ? { currentTool } : {}),
|
|
2229
|
+
usage: { toolUses: toolStarts },
|
|
2230
|
+
});
|
|
2231
|
+
});
|
|
2148
2232
|
const bgInternals = bgSink
|
|
2149
2233
|
? {
|
|
2150
2234
|
...childInternals,
|
|
@@ -2155,6 +2239,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
2155
2239
|
}
|
|
2156
2240
|
catch {
|
|
2157
2241
|
}
|
|
2242
|
+
bgActivityBeat(e);
|
|
2158
2243
|
if (e.type === "task_progress") {
|
|
2159
2244
|
const currentAction = stepRecorder.currentAction();
|
|
2160
2245
|
const currentTool = stepRecorder.currentActionStructured();
|
|
@@ -2388,6 +2473,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
2388
2473
|
? classifySubagentError({ status: "failed", ...(child.errorCode !== undefined ? { errorCode: child.errorCode } : {}), ...(child.errorMessage !== undefined ? { errorMessage: child.errorMessage } : {}) })
|
|
2389
2474
|
: undefined;
|
|
2390
2475
|
const settled = bg.registry.settleBackgroundAgent(taskId, {
|
|
2476
|
+
cycle: 0,
|
|
2391
2477
|
status: ok ? "completed" : reaped ? "killed" : "failed",
|
|
2392
2478
|
seq: seqAtSettle ?? 1,
|
|
2393
2479
|
...resultSettleFields(child.result),
|
|
@@ -2484,6 +2570,7 @@ function createSubagentToolNode(opts, depth, excluded, extraToolsBudget) {
|
|
|
2484
2570
|
const errCodeBgReject = killed ? undefined : extractErrorCode(msg);
|
|
2485
2571
|
const errClassBgReject = killed ? undefined : classifySubagentError({ status: "failed", errorMessage: msg });
|
|
2486
2572
|
const settled = bg.registry.settleBackgroundAgent(taskId, {
|
|
2573
|
+
cycle: 0,
|
|
2487
2574
|
status: killed ? "killed" : "failed",
|
|
2488
2575
|
error: msg,
|
|
2489
2576
|
seq: seqAtSettle ?? 1,
|
|
@@ -42,20 +42,23 @@ export function createCircuitBreakerBrain(inner, opts = {}) {
|
|
|
42
42
|
const tryPass = (key) => {
|
|
43
43
|
const s = snap(key);
|
|
44
44
|
if (s.phase === "closed")
|
|
45
|
-
return true;
|
|
45
|
+
return { pass: true };
|
|
46
46
|
if (s.phase === "open") {
|
|
47
|
-
if (s.openedAt
|
|
47
|
+
if (s.openedAt === undefined)
|
|
48
|
+
return { pass: false };
|
|
49
|
+
const elapsed = now() - s.openedAt;
|
|
50
|
+
if (elapsed >= cooldownMs) {
|
|
48
51
|
setState(key, s.phase, { ...s, phase: "half-open", halfOpenInFlight: 1 });
|
|
49
|
-
return true;
|
|
52
|
+
return { pass: true };
|
|
50
53
|
}
|
|
51
|
-
return false;
|
|
54
|
+
return { pass: false, retryInMs: cooldownMs - elapsed };
|
|
52
55
|
}
|
|
53
56
|
const inFlight = s.halfOpenInFlight ?? 0;
|
|
54
57
|
if (inFlight < halfOpenProbes) {
|
|
55
58
|
state.set(key, { ...s, halfOpenInFlight: inFlight + 1 });
|
|
56
|
-
return true;
|
|
59
|
+
return { pass: true };
|
|
57
60
|
}
|
|
58
|
-
return false;
|
|
61
|
+
return { pass: false };
|
|
59
62
|
};
|
|
60
63
|
const record = (key, failedCode, viaProbe = true) => {
|
|
61
64
|
const s = snap(key);
|
|
@@ -81,8 +84,15 @@ export function createCircuitBreakerBrain(inner, opts = {}) {
|
|
|
81
84
|
const key = keyOf(model);
|
|
82
85
|
void (async () => {
|
|
83
86
|
const bypass = options?.resilience?.bypassBreaker === true;
|
|
84
|
-
|
|
85
|
-
|
|
87
|
+
const decision = bypass ? { pass: true } : tryPass(key);
|
|
88
|
+
if (!decision.pass) {
|
|
89
|
+
emitBrainStatus({
|
|
90
|
+
phase: "circuit_open",
|
|
91
|
+
detail: "repeated errors, pausing briefly",
|
|
92
|
+
...(decision.retryInMs !== undefined
|
|
93
|
+
? { retryInMs: decision.retryInMs, retryInSec: Math.ceil(decision.retryInMs / 1000) }
|
|
94
|
+
: {}),
|
|
95
|
+
});
|
|
86
96
|
out.push({ type: "error", reason: "error", error: errorAssistantMessage(model, "network", `${CIRCUIT_OPEN_MARKER}${key}`) });
|
|
87
97
|
return;
|
|
88
98
|
}
|
package/dist/brain/retry.d.ts
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export declare function parseRetryAfter(res: Response | undefined): number | undefined;
|
|
2
|
+
export declare function parseRateLimitReset(res: Response | undefined): number | undefined;
|
|
2
3
|
export declare function retryBackoffMs(baseDelayMs: number, attempt: number, res?: Response, rand?: () => number): number;
|
package/dist/brain/retry.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
const MAX_BACKOFF_MS =
|
|
2
|
-
const
|
|
1
|
+
const MAX_BACKOFF_MS = 32_000;
|
|
2
|
+
const JITTER_FRACTION = 0.25;
|
|
3
|
+
const MAX_HEADER_WAIT_MS = 60_000;
|
|
4
|
+
const RATE_LIMIT_STATUS = 429;
|
|
3
5
|
export function parseRetryAfter(res) {
|
|
4
6
|
const h = res?.headers?.get?.("retry-after");
|
|
5
7
|
if (!h)
|
|
@@ -12,11 +14,31 @@ export function parseRetryAfter(res) {
|
|
|
12
14
|
return Math.max(0, date - Date.now());
|
|
13
15
|
return undefined;
|
|
14
16
|
}
|
|
17
|
+
export function parseRateLimitReset(res) {
|
|
18
|
+
const h = res?.headers?.get?.("anthropic-ratelimit-unified-reset");
|
|
19
|
+
if (!h)
|
|
20
|
+
return undefined;
|
|
21
|
+
const epochSecs = Number(h);
|
|
22
|
+
if (!Number.isFinite(epochSecs))
|
|
23
|
+
return undefined;
|
|
24
|
+
const ms = Math.round(epochSecs * 1000 - Date.now());
|
|
25
|
+
return ms > 0 ? ms : undefined;
|
|
26
|
+
}
|
|
27
|
+
function providerWaitHintMs(res) {
|
|
28
|
+
const hints = [];
|
|
29
|
+
const retryAfter = parseRetryAfter(res);
|
|
30
|
+
if (retryAfter !== undefined)
|
|
31
|
+
hints.push(Math.min(MAX_HEADER_WAIT_MS, retryAfter));
|
|
32
|
+
if (res?.status === RATE_LIMIT_STATUS) {
|
|
33
|
+
const reset = parseRateLimitReset(res);
|
|
34
|
+
if (reset !== undefined)
|
|
35
|
+
hints.push(Math.min(MAX_HEADER_WAIT_MS, reset));
|
|
36
|
+
}
|
|
37
|
+
return hints.length === 0 ? undefined : Math.max(...hints);
|
|
38
|
+
}
|
|
15
39
|
export function retryBackoffMs(baseDelayMs, attempt, res, rand = Math.random) {
|
|
16
40
|
const exp = Math.min(MAX_BACKOFF_MS, baseDelayMs * 2 ** attempt);
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
return jittered;
|
|
21
|
-
return Math.max(Math.min(MAX_RETRY_AFTER_MS, retryAfter), jittered);
|
|
41
|
+
const computed = Math.round(exp + rand() * JITTER_FRACTION * exp);
|
|
42
|
+
const hint = providerWaitHintMs(res);
|
|
43
|
+
return hint === undefined ? computed : Math.max(hint, computed);
|
|
22
44
|
}
|
|
@@ -4,6 +4,7 @@ export interface StreamEngineConfig extends BrainTimeoutConfig {
|
|
|
4
4
|
maxRetries?: number;
|
|
5
5
|
retryDelayMs?: number;
|
|
6
6
|
}
|
|
7
|
+
export declare function resolveMaxRetries(configured: number | undefined): number;
|
|
7
8
|
export interface SSERequest {
|
|
8
9
|
url: string;
|
|
9
10
|
headers: Record<string, string>;
|
|
@@ -4,6 +4,22 @@ import { retryBackoffMs } from "./retry.js";
|
|
|
4
4
|
import { emitBrainStatus, emitBrainTelemetry } from "./status-sink.js";
|
|
5
5
|
import { createConnectController } from "./timeout.js";
|
|
6
6
|
import { WALLTIME_CUTOFF_MESSAGE } from "./walltime.js";
|
|
7
|
+
const DEFAULT_MAX_RETRIES = 10;
|
|
8
|
+
const MAX_RETRIES_ENV_CEILING = 15;
|
|
9
|
+
const MAX_RETRIES_ENV = "SEMA_MAX_RETRIES";
|
|
10
|
+
export function resolveMaxRetries(configured) {
|
|
11
|
+
if (configured !== undefined)
|
|
12
|
+
return configured;
|
|
13
|
+
const raw = process.env[MAX_RETRIES_ENV];
|
|
14
|
+
if (raw !== undefined && raw.trim() !== "") {
|
|
15
|
+
const parsed = Number(raw);
|
|
16
|
+
if (Number.isFinite(parsed) && parsed >= 0)
|
|
17
|
+
return Math.min(Math.floor(parsed), MAX_RETRIES_ENV_CEILING);
|
|
18
|
+
}
|
|
19
|
+
return DEFAULT_MAX_RETRIES;
|
|
20
|
+
}
|
|
21
|
+
const RETRY_STATUS_SLICE_MS = 30_000;
|
|
22
|
+
const THINKING_RETRY_BUDGET = 2;
|
|
7
23
|
const EMPTY_USAGE = {
|
|
8
24
|
input: 0,
|
|
9
25
|
output: 0,
|
|
@@ -43,25 +59,54 @@ function sleep(ms, signal) {
|
|
|
43
59
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
44
60
|
});
|
|
45
61
|
}
|
|
62
|
+
async function sleepAnnouncingRetry(totalMs, signal, frame) {
|
|
63
|
+
let remaining = Math.max(0, totalMs);
|
|
64
|
+
for (;;) {
|
|
65
|
+
if (signal?.aborted)
|
|
66
|
+
return;
|
|
67
|
+
emitBrainStatus(frame(remaining));
|
|
68
|
+
if (remaining <= 0)
|
|
69
|
+
return;
|
|
70
|
+
const slice = Math.min(remaining, RETRY_STATUS_SLICE_MS);
|
|
71
|
+
await sleep(slice, signal);
|
|
72
|
+
remaining -= slice;
|
|
73
|
+
if (remaining <= 0)
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
46
77
|
export function runStreamingBrain(args) {
|
|
47
78
|
const { model, doFetch, signal, config, httpLabel, buildRequest, makeParser, callDeadlineMs, stallTimeouts } = args;
|
|
48
79
|
const out = createAssistantMessageEventStream();
|
|
49
80
|
let cleanup;
|
|
81
|
+
let announcedRetry = false;
|
|
82
|
+
let terminalRetryPhase = "recovered";
|
|
83
|
+
let terminalRetryDetail = "recovered after retrying";
|
|
50
84
|
void run()
|
|
51
85
|
.catch((err) => {
|
|
52
86
|
const aborted = signal?.aborted === true || isAbortError(err);
|
|
87
|
+
terminalRetryPhase = "gave_up";
|
|
88
|
+
terminalRetryDetail = aborted ? "cancelled while retrying" : "retries exhausted";
|
|
53
89
|
const errorMsg = emptyAssistant(model);
|
|
54
90
|
errorMsg.stopReason = aborted ? "aborted" : "error";
|
|
55
91
|
errorMsg.errorMessage = err instanceof Error ? err.message : String(err);
|
|
56
92
|
errorMsg.usageMissing = true;
|
|
57
93
|
out.push({ type: "error", reason: aborted ? "aborted" : "error", error: errorMsg });
|
|
58
94
|
})
|
|
59
|
-
.finally(() =>
|
|
95
|
+
.finally(() => {
|
|
96
|
+
cleanup?.();
|
|
97
|
+
if (!announcedRetry)
|
|
98
|
+
return;
|
|
99
|
+
try {
|
|
100
|
+
emitBrainStatus({ phase: terminalRetryPhase, detail: terminalRetryDetail });
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
}
|
|
104
|
+
});
|
|
60
105
|
return out;
|
|
61
106
|
async function run() {
|
|
62
107
|
const req = buildRequest();
|
|
63
|
-
const maxRetries = config.maxRetries
|
|
64
|
-
const baseDelay = config.retryDelayMs ??
|
|
108
|
+
const maxRetries = resolveMaxRetries(config.maxRetries);
|
|
109
|
+
const baseDelay = config.retryDelayMs ?? 500;
|
|
65
110
|
const firstTokenTimeoutMs = config.firstTokenTimeoutMs ?? stallTimeouts?.firstTokenMs;
|
|
66
111
|
const idleTimeoutMs = config.idleTimeoutMs ?? stallTimeouts?.idleMs;
|
|
67
112
|
const connectTimeoutMs = config.connectTimeoutMs ?? stallTimeouts?.connectMs;
|
|
@@ -126,16 +171,19 @@ export function runStreamingBrain(args) {
|
|
|
126
171
|
}
|
|
127
172
|
const statusPhase = r?.status === 429 ? "rate_limited" : netErr !== undefined ? "reconnecting" : "retrying";
|
|
128
173
|
emitBrainTelemetry({ kind: "retry", attempt: attempt + 1, phase: "connect" });
|
|
129
|
-
|
|
174
|
+
announcedRetry = true;
|
|
175
|
+
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
|
|
130
176
|
phase: statusPhase,
|
|
131
177
|
detail: statusPhase === "rate_limited"
|
|
132
178
|
? "rate limited, backing off"
|
|
133
179
|
: statusPhase === "reconnecting"
|
|
134
180
|
? "connection lost, reconnecting"
|
|
135
181
|
: "transient error, retrying",
|
|
136
|
-
retryInSec: Math.ceil(
|
|
137
|
-
|
|
138
|
-
|
|
182
|
+
retryInSec: Math.ceil(remainingMs / 1000),
|
|
183
|
+
retryInMs: remainingMs,
|
|
184
|
+
attempt: attempt + 1,
|
|
185
|
+
maxRetries,
|
|
186
|
+
}));
|
|
139
187
|
continue;
|
|
140
188
|
}
|
|
141
189
|
if (netErr)
|
|
@@ -311,14 +359,21 @@ export function runStreamingBrain(args) {
|
|
|
311
359
|
cleanup?.();
|
|
312
360
|
cleanup = undefined;
|
|
313
361
|
if (snap.hasOnlyThinking) {
|
|
314
|
-
if (thinkingRetries >=
|
|
362
|
+
if (thinkingRetries >= THINKING_RETRY_BUDGET)
|
|
315
363
|
throw failure.err;
|
|
316
364
|
thinkingRetries++;
|
|
317
365
|
parser.sealForRetry();
|
|
318
366
|
const delayMs = 100 * thinkingRetries;
|
|
319
367
|
emitBrainTelemetry({ kind: "retry", attempt: thinkingRetries, phase: "midstream" });
|
|
320
|
-
|
|
321
|
-
await
|
|
368
|
+
announcedRetry = true;
|
|
369
|
+
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
|
|
370
|
+
phase: "reconnecting",
|
|
371
|
+
detail: "connection lost, reconnecting",
|
|
372
|
+
retryInSec: Math.ceil(remainingMs / 1000),
|
|
373
|
+
retryInMs: remainingMs,
|
|
374
|
+
attempt: thinkingRetries,
|
|
375
|
+
maxRetries: THINKING_RETRY_BUDGET,
|
|
376
|
+
}));
|
|
322
377
|
continue;
|
|
323
378
|
}
|
|
324
379
|
if (attempt >= maxRetries)
|
|
@@ -326,8 +381,15 @@ export function runStreamingBrain(args) {
|
|
|
326
381
|
const delayMs = retryBackoffMs(baseDelay, attempt);
|
|
327
382
|
attempt++;
|
|
328
383
|
emitBrainTelemetry({ kind: "retry", attempt, phase: "midstream" });
|
|
329
|
-
|
|
330
|
-
await
|
|
384
|
+
announcedRetry = true;
|
|
385
|
+
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
|
|
386
|
+
phase: "reconnecting",
|
|
387
|
+
detail: "connection lost, reconnecting",
|
|
388
|
+
retryInSec: Math.ceil(remainingMs / 1000),
|
|
389
|
+
retryInMs: remainingMs,
|
|
390
|
+
attempt,
|
|
391
|
+
maxRetries,
|
|
392
|
+
}));
|
|
331
393
|
}
|
|
332
394
|
}
|
|
333
395
|
}
|
|
@@ -4,3 +4,4 @@ export declare const SESSION_BG_DEFAULT_TIMEOUT_SEC: number;
|
|
|
4
4
|
export declare const RETAIN_DEFAULT_TTL_MS: number;
|
|
5
5
|
export declare const RETAIN_DEFAULT_MAX = 16;
|
|
6
6
|
export declare const SESSION_DEFAULT_TTL_DAYS = 7;
|
|
7
|
+
export declare const RUNNING_AGENT_OBSERVE_EVERY_BEATS = 4;
|
package/dist/config/defaults.js
CHANGED