@sema-agent/core 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/observer.d.ts +14 -0
- package/dist/agents/observer.js +58 -9
- package/dist/agents/send-message-tool.js +55 -10
- package/dist/agents/subagent.d.ts +1 -0
- package/dist/agents/subagent.js +69 -15
- package/dist/agents/teacher.js +51 -23
- package/dist/core/context-edit.js +16 -3
- package/dist/core/file-snapshot-store.js +10 -1
- package/dist/core/runner/assemble-result.d.ts +1 -0
- package/dist/core/runner/assemble-result.js +12 -8
- package/dist/core/runner/prepare-task.d.ts +1 -0
- package/dist/core/runner/prepare-task.js +40 -10
- package/dist/core/runner/runtask.js +27 -5
- package/dist/core/runner/synthetic-tools.d.ts +1 -0
- package/dist/core/runner/synthetic-tools.js +18 -15
- package/dist/core/runner/turn-attachments.d.ts +12 -2
- package/dist/core/runner/turn-attachments.js +33 -3
- package/dist/core/task-registry-agent.d.ts +11 -1
- package/dist/core/task-registry-agent.js +40 -3
- package/dist/core/task-registry-monitor.js +149 -29
- package/dist/core/task-registry-shared.d.ts +25 -2
- package/dist/core/task-registry-shared.js +26 -2
- package/dist/core/task-registry.d.ts +5 -0
- package/dist/core/task-registry.js +25 -26
- package/dist/core/tools.d.ts +2 -0
- package/dist/core/tools.js +9 -0
- package/dist/core/types.d.ts +3 -1
- package/dist/core/workflow-journal-store.d.ts +16 -0
- package/dist/core/workflow-journal-store.js +28 -0
- package/dist/engine/session/memory-repo.js +5 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/orchestration/workflow-size-guideline.d.ts +6 -1
- package/dist/orchestration/workflow-size-guideline.js +19 -9
- package/dist/orchestration/workflow.d.ts +16 -0
- package/dist/orchestration/workflow.js +80 -20
- package/dist/prompt-assembly/assemble.js +3 -7
- package/dist/prompt-assembly/packs/sema-default.js +8 -5
- package/dist/prompts/coordinator.d.ts +1 -1
- package/dist/prompts/coordinator.js +45 -0
- package/dist/prompts/default.d.ts +4 -5
- package/dist/prompts/default.js +16 -18
- package/dist/prompts/simple-sections.d.ts +3 -1
- package/dist/prompts/simple-sections.js +11 -1
- package/dist/stores/file/workflow-journal-store.d.ts +7 -1
- package/dist/stores/file/workflow-journal-store.js +70 -33
- package/dist/tools/fs/bash-readonly-classifier.js +20 -1
- package/dist/tools/fs/fs-bash.d.ts +1 -0
- package/dist/tools/fs/fs-bash.js +10 -3
- package/dist/tools/fs/fs-read.js +10 -10
- package/dist/tools/fs/fs-search-tools.js +42 -7
- package/dist/tools/fs/fs-write.js +18 -6
- package/dist/tools/fs/index.d.ts +1 -0
- package/dist/tools/fs/index.js +2 -1
- package/dist/tools/fs/safety.d.ts +4 -0
- package/dist/tools/fs/safety.js +102 -6
- package/dist/tools/fs/search.d.ts +1 -0
- package/dist/tools/fs/search.js +23 -3
- package/dist/tools/monitor.d.ts +2 -0
- package/dist/tools/monitor.js +3 -1
- package/package.json +3 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { delimitUntrusted } from "./untrusted-text.js";
|
|
2
|
-
import { assertOwnership, defaultMonitorTimers, MONITOR_MAX_TIMEOUT_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_BATCHES_PER_MINUTE, TASK_OUTPUT_MAX_CHARS, mintCompletionId, clipMonitorEvent, accountDroppedBytes, rollSpoolText, statusFromBackground, droppedGapNote, firstDropNote, alreadyTerminalStopNote, clipTaskOutput, sleepPollStep, } from "./task-registry-shared.js";
|
|
2
|
+
import { assertOwnership, defaultMonitorTimers, MONITOR_MAX_TIMEOUT_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_BATCHES_PER_MINUTE, MONITOR_STORM_BURST, MONITOR_STORM_KILL_AFTER_MS, MONITOR_LINE_BUF_CAP, MONITOR_SPILL_CAP_BYTES, TASK_OUTPUT_MAX_CHARS, mintCompletionId, clipMonitorEvent, clipMonitorLine, terminalTaskSummary, accountDroppedBytes, rollSpoolText, statusFromBackground, droppedGapNote, firstDropNote, alreadyTerminalStopNote, clipTaskOutput, sleepPollStep, } from "./task-registry-shared.js";
|
|
3
|
+
import { buildToolResultRef, OFFLOAD_TOOL_NAME } from "./tool-result-store.js";
|
|
3
4
|
export function registerMonitorLane(core, input) {
|
|
4
5
|
assertOwnership(input, "registerMonitor");
|
|
5
6
|
const id = core.mintTaskId("monitor");
|
|
@@ -9,6 +10,8 @@ export function registerMonitorLane(core, input) {
|
|
|
9
10
|
const timeoutMs = persistent
|
|
10
11
|
? undefined
|
|
11
12
|
: Math.min(MONITOR_MAX_TIMEOUT_MS, Math.max(1, Math.floor(input.timeoutMs ?? MONITOR_DEFAULT_TIMEOUT_MS)));
|
|
13
|
+
const maxBatchesPerMinute = Math.max(1, Math.floor(input.maxBatchesPerMinute ?? MONITOR_MAX_BATCHES_PER_MINUTE));
|
|
14
|
+
const stormBurst = Math.max(1, Math.floor(input.stormBurst ?? MONITOR_STORM_BURST));
|
|
12
15
|
const handle = {
|
|
13
16
|
id,
|
|
14
17
|
type: "monitor",
|
|
@@ -26,28 +29,93 @@ export function registerMonitorLane(core, input) {
|
|
|
26
29
|
spool: { stdout: "", stderr: "", rolledChars: 0 },
|
|
27
30
|
lineBuf: "",
|
|
28
31
|
seq: 0,
|
|
29
|
-
|
|
32
|
+
stormTokens: stormBurst,
|
|
33
|
+
stormBurst,
|
|
34
|
+
stormRefillIntervalMs: Math.max(1, Math.round(60_000 / maxBatchesPerMinute)),
|
|
35
|
+
stormRefilledAt: timers.now(),
|
|
30
36
|
batchWindowMs: Math.max(1, Math.floor(input.batchWindowMs ?? MONITOR_BATCH_WINDOW_MS)),
|
|
31
|
-
maxBatchesPerMinute
|
|
37
|
+
maxBatchesPerMinute,
|
|
32
38
|
...(timeoutMs !== undefined ? { timeoutMs, deadlineAt: timers.now() + timeoutMs } : {}),
|
|
33
39
|
timers,
|
|
34
40
|
...(input.onEvent !== undefined ? { onEvent: input.onEvent } : {}),
|
|
41
|
+
...(input.toolResultStore !== undefined ? { toolResultStore: input.toolResultStore } : {}),
|
|
42
|
+
...(input.sessionId !== undefined ? { spillSessionId: input.sessionId } : {}),
|
|
35
43
|
};
|
|
36
44
|
core.handles.set(id, handle);
|
|
37
45
|
startMonitorWatcherLane(core, id);
|
|
38
46
|
return id;
|
|
39
47
|
}
|
|
48
|
+
function spillRolledMonitorChunk(handle, stream, dropped) {
|
|
49
|
+
const store = handle.toolResultStore;
|
|
50
|
+
if (store === undefined)
|
|
51
|
+
return;
|
|
52
|
+
const used = handle.spillBytesUsed ?? 0;
|
|
53
|
+
if (used >= MONITOR_SPILL_CAP_BYTES) {
|
|
54
|
+
handle.spillCapped = true;
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const n = stream === "out" ? (handle.spillSegCount ?? 0) : (handle.spillErrSegCount ?? 0);
|
|
58
|
+
const ref = buildToolResultRef(handle.spillSessionId ?? "no-session", `${handle.id}_${stream}_seg${n}`);
|
|
59
|
+
handle.spillBytesUsed = used + dropped.length;
|
|
60
|
+
if (stream === "out")
|
|
61
|
+
handle.spillSegCount = n + 1;
|
|
62
|
+
else
|
|
63
|
+
handle.spillErrSegCount = n + 1;
|
|
64
|
+
try {
|
|
65
|
+
void Promise.resolve(store.put(ref, dropped)).catch(() => {
|
|
66
|
+
handle.spillFailed = true;
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
handle.spillFailed = true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function monitorSpillNote(handle) {
|
|
74
|
+
const outN = handle.spillSegCount ?? 0;
|
|
75
|
+
const errN = handle.spillErrSegCount ?? 0;
|
|
76
|
+
if (outN === 0 && errN === 0)
|
|
77
|
+
return "";
|
|
78
|
+
const sid = handle.spillSessionId ?? "no-session";
|
|
79
|
+
const segLabel = (stream, n) => {
|
|
80
|
+
const first = buildToolResultRef(sid, `${handle.id}_${stream}_seg0`);
|
|
81
|
+
return n <= 1 ? `ref "${first}"` : `refs "${first}" .. "${buildToolResultRef(sid, `${handle.id}_${stream}_seg${n - 1}`)}"`;
|
|
82
|
+
};
|
|
83
|
+
const clauses = [];
|
|
84
|
+
if (outN > 0)
|
|
85
|
+
clauses.push(`stdout ${segLabel("out", outN)}`);
|
|
86
|
+
if (errN > 0)
|
|
87
|
+
clauses.push(`stderr ${segLabel("err", errN)}`);
|
|
88
|
+
const coverage = handle.spillFailed === true
|
|
89
|
+
? " — a write failed partway through; the ref chain may be INCOMPLETE, read what is there"
|
|
90
|
+
: handle.spillCapped === true
|
|
91
|
+
? ` — spill cap (${MONITOR_SPILL_CAP_BYTES} bytes) reached; earlier segments retained, later rolls were not spilled`
|
|
92
|
+
: "";
|
|
93
|
+
return `; spilled to ${clauses.join(", ")} (read back via ${OFFLOAD_TOOL_NAME})${coverage}`;
|
|
94
|
+
}
|
|
40
95
|
export function absorbMonitorPollLane(handle, v) {
|
|
41
96
|
const spool = handle.spool;
|
|
42
97
|
accountDroppedBytes(spool, v);
|
|
43
98
|
const ROLL_CAP = 2 * TASK_OUTPUT_MAX_CHARS;
|
|
44
|
-
spool.stdout = rollSpoolText(spool, spool.stdout + v.stdout, ROLL_CAP);
|
|
45
|
-
spool.stderr = rollSpoolText(spool, spool.stderr + v.stderr, ROLL_CAP);
|
|
99
|
+
spool.stdout = rollSpoolText(spool, spool.stdout + v.stdout, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "out", dropped));
|
|
100
|
+
spool.stderr = rollSpoolText(spool, spool.stderr + v.stderr, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "err", dropped));
|
|
46
101
|
handle.lineBuf += v.stdout;
|
|
102
|
+
if (handle.lineBuf.length > MONITOR_LINE_BUF_CAP)
|
|
103
|
+
handle.lineBuf = handle.lineBuf.slice(-MONITOR_LINE_BUF_CAP);
|
|
47
104
|
const parts = handle.lineBuf.split(/\r?\n/);
|
|
48
105
|
handle.lineBuf = parts.pop() ?? "";
|
|
49
106
|
return parts.filter((l) => l !== "");
|
|
50
107
|
}
|
|
108
|
+
function consumeStormToken(handle, now) {
|
|
109
|
+
const gained = Math.floor((now - handle.stormRefilledAt) / handle.stormRefillIntervalMs);
|
|
110
|
+
if (gained > 0) {
|
|
111
|
+
handle.stormTokens = Math.min(handle.stormBurst, handle.stormTokens + gained);
|
|
112
|
+
handle.stormRefilledAt += gained * handle.stormRefillIntervalMs;
|
|
113
|
+
}
|
|
114
|
+
if (handle.stormTokens <= 0)
|
|
115
|
+
return false;
|
|
116
|
+
handle.stormTokens -= 1;
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
51
119
|
export function emitMonitorEventLane(handle, n) {
|
|
52
120
|
try {
|
|
53
121
|
handle.onEvent?.(n);
|
|
@@ -75,7 +143,8 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
75
143
|
handle.lineBuf = "";
|
|
76
144
|
return out;
|
|
77
145
|
};
|
|
78
|
-
const
|
|
146
|
+
const pendingSuppressionNote = () => consumePendingSuppressionNote(handle);
|
|
147
|
+
const terminal = (status, detail, finalLines, exitCode) => {
|
|
79
148
|
handle.updatedAt = Date.now();
|
|
80
149
|
mintCompletionId(handle);
|
|
81
150
|
stop();
|
|
@@ -85,8 +154,8 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
85
154
|
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
86
155
|
status,
|
|
87
156
|
...(status === "killed" && handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
|
|
88
|
-
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${
|
|
89
|
-
...(finalLines.length > 0 ? { lines: finalLines, result: clipMonitorEvent(finalLines.join("\n")), ...(status === "killed" ? { partial: true } : {}) } : {}),
|
|
157
|
+
summary: `${terminalTaskSummary("monitor", (handle.description ?? "monitor").slice(0, 200), status, exitCode)} — ${detail}${droppedGapNote(handle.spool)}`,
|
|
158
|
+
...(finalLines.length > 0 ? { lines: finalLines.map(clipMonitorLine), result: clipMonitorEvent(finalLines.map(clipMonitorLine).join("\n")), ...(status === "killed" ? { partial: true } : {}) } : {}),
|
|
90
159
|
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
|
|
91
160
|
}));
|
|
92
161
|
};
|
|
@@ -107,7 +176,7 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
107
176
|
if ((r.error.code === "timeout" || r.error.code === "io") && ++pollFailures <= MAX_RETRYABLE_POLL_FAILURES)
|
|
108
177
|
return;
|
|
109
178
|
handle.status = "failed";
|
|
110
|
-
return terminal("failed", `monitor lost its process (${r.error.message}); watch ended
|
|
179
|
+
return terminal("failed", `monitor lost its process (${r.error.message}); watch ended.${pendingSuppressionNote()}`, pendingFinalLines([]));
|
|
111
180
|
}
|
|
112
181
|
pollFailures = 0;
|
|
113
182
|
const spool = handle.spool;
|
|
@@ -127,7 +196,7 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
127
196
|
: r.value.timedOut === true
|
|
128
197
|
? `timed out after ${r.value.timeoutSec !== undefined ? `${r.value.timeoutSec}s` : "its background time budget"} (the execution environment's background time budget); the process was killed and the watch ended.`
|
|
129
198
|
: `${r.value.status}; watch ended.`;
|
|
130
|
-
return terminal(handle.status === "completed" ? "completed" : handle.status === "killed" ? "killed" : "failed", label
|
|
199
|
+
return terminal(handle.status === "completed" ? "completed" : handle.status === "killed" ? "killed" : "failed", `${label}${pendingSuppressionNote()}`, pendingFinalLines(lines), r.value.status === "exited" ? r.value.exitCode : undefined);
|
|
131
200
|
}
|
|
132
201
|
const now = timers.now();
|
|
133
202
|
if (handle.deadlineAt !== undefined && now >= handle.deadlineAt) {
|
|
@@ -137,45 +206,74 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
137
206
|
return stop();
|
|
138
207
|
if (!k.ok && k.error.code !== "not_found") {
|
|
139
208
|
handle.status = "failed";
|
|
140
|
-
return terminal("failed", `timed out after ${handle.timeoutMs}ms, but the kill failed (${k.error.message}); the process may still be running. Watch ended
|
|
209
|
+
return terminal("failed", `timed out after ${handle.timeoutMs}ms, but the kill failed (${k.error.message}); the process may still be running. Watch ended.${pendingSuppressionNote()}`, pendingFinalLines(lines));
|
|
141
210
|
}
|
|
142
211
|
handle.status = "killed";
|
|
143
212
|
handle.stoppedBy = handle.stopSource ?? "timeout";
|
|
144
|
-
return terminal("killed", `timed out after ${handle.timeoutMs}ms; the process was killed and the watch ended
|
|
213
|
+
return terminal("killed", `timed out after ${handle.timeoutMs}ms; the process was killed and the watch ended.${pendingSuppressionNote()}`, pendingFinalLines(lines));
|
|
145
214
|
}
|
|
146
215
|
if (lines.length === 0) {
|
|
147
216
|
if (r.value.stdout.length > 0 || r.value.stderr.length > 0)
|
|
148
217
|
handle.updatedAt = Date.now();
|
|
149
218
|
return;
|
|
150
219
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
220
|
+
if (!consumeStormToken(handle, now)) {
|
|
221
|
+
const previousSuppressedAt = handle.lastSuppressedAt;
|
|
222
|
+
handle.suppressedBatches = (handle.suppressedBatches ?? 0) + 1;
|
|
223
|
+
handle.lastSuppressedAt = now;
|
|
224
|
+
if (previousSuppressedAt !== undefined && now - previousSuppressedAt > handle.stormRefillIntervalMs * 3) {
|
|
225
|
+
handle.overloadSince = now;
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
handle.overloadSince ??= now;
|
|
229
|
+
}
|
|
230
|
+
handle.updatedAt = Date.now();
|
|
231
|
+
if (now - handle.overloadSince <= MONITOR_STORM_KILL_AFTER_MS)
|
|
232
|
+
return;
|
|
233
|
+
const suppressed = handle.suppressedBatches;
|
|
234
|
+
const overloadSecs = Math.round((now - handle.overloadSince) / 1000);
|
|
155
235
|
core.markStopSource(id, "system");
|
|
156
236
|
const k = await handle.env.killBackground(handle.shellId);
|
|
157
237
|
if (core.handles.get(id) !== handle || handle.status !== "running")
|
|
158
238
|
return stop();
|
|
159
239
|
if (!k.ok && k.error.code !== "not_found") {
|
|
160
240
|
handle.status = "failed";
|
|
161
|
-
return terminal("failed", `auto-stop triggered (
|
|
241
|
+
return terminal("failed", `auto-stop triggered (${suppressed} event batch(es) suppressed over ${overloadSecs}s of sustained overload), but the kill failed (${k.error.message}); the process may still be running. Watch ended.`, pendingFinalLines(lines));
|
|
162
242
|
}
|
|
163
243
|
handle.status = "killed";
|
|
164
244
|
handle.stoppedBy = handle.stopSource ?? "system";
|
|
165
|
-
return terminal("killed", `auto-stopped:
|
|
245
|
+
return terminal("killed", `auto-stopped: too much output (${suppressed} event batch(es) suppressed over ${overloadSecs}s of sustained overload; sustained budget is ${handle.maxBatchesPerMinute} batches/minute). ` +
|
|
166
246
|
`The process was killed; its full output remains readable via TaskOutput. Restart it with a quieter command (filter its output) if you still need the watch.`, pendingFinalLines(lines));
|
|
167
247
|
}
|
|
248
|
+
if ((handle.suppressedBatches ?? 0) > 0) {
|
|
249
|
+
const suppressed = handle.suppressedBatches;
|
|
250
|
+
handle.suppressedBatches = 0;
|
|
251
|
+
if (handle.lastSuppressedAt !== undefined && now - handle.lastSuppressedAt > handle.stormRefillIntervalMs * 3)
|
|
252
|
+
handle.overloadSince = undefined;
|
|
253
|
+
handle.seq += 1;
|
|
254
|
+
const note = `[${suppressed} events suppressed — output rate too high. Consider using TaskStop to restart this monitor with a more selective filter.]`;
|
|
255
|
+
emit({
|
|
256
|
+
task_id: id,
|
|
257
|
+
task_type: "monitor",
|
|
258
|
+
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
259
|
+
status: "event",
|
|
260
|
+
seq: handle.seq,
|
|
261
|
+
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${suppressed} event batch(es) suppressed (output rate too high); the suppressed output is still readable via TaskOutput`,
|
|
262
|
+
result: note,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
168
265
|
handle.seq += 1;
|
|
169
266
|
handle.updatedAt = Date.now();
|
|
267
|
+
const shown = lines.map(clipMonitorLine);
|
|
170
268
|
emit({
|
|
171
269
|
task_id: id,
|
|
172
270
|
task_type: "monitor",
|
|
173
271
|
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
174
272
|
status: "event",
|
|
175
273
|
seq: handle.seq,
|
|
176
|
-
lines,
|
|
274
|
+
lines: shown,
|
|
177
275
|
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${lines.length} new output line(s)${firstDropNote(spool)}`,
|
|
178
|
-
result: clipMonitorEvent(
|
|
276
|
+
result: clipMonitorEvent(shown.join("\n")),
|
|
179
277
|
});
|
|
180
278
|
}
|
|
181
279
|
catch (e) {
|
|
@@ -184,7 +282,7 @@ export function startMonitorWatcherLane(core, id) {
|
|
|
184
282
|
if (++pollFailures <= MAX_RETRYABLE_POLL_FAILURES)
|
|
185
283
|
return;
|
|
186
284
|
handle.status = "failed";
|
|
187
|
-
terminal("failed", `monitor watcher crashed (${e instanceof Error ? e.message : String(e)}); watch ended
|
|
285
|
+
terminal("failed", `monitor watcher crashed (${e instanceof Error ? e.message : String(e)}); watch ended.${pendingSuppressionNote()}`, pendingFinalLines([]));
|
|
188
286
|
}
|
|
189
287
|
finally {
|
|
190
288
|
ticking = false;
|
|
@@ -216,7 +314,7 @@ export async function pollMonitorLane(handle, filter, deadline, signal) {
|
|
|
216
314
|
const retrieval = running && deadline !== undefined && Date.now() >= deadline ? "timeout" : "success";
|
|
217
315
|
const timedOutClause = spool.timedOut === true ? ` (timed out after ${spool.timeoutSec !== undefined ? `${spool.timeoutSec}s` : "its background time budget"})` : "";
|
|
218
316
|
const displayStatus = (spool.exitCode !== undefined ? `exited(code ${spool.exitCode})` : running ? "running" : handle.status) + timedOutClause;
|
|
219
|
-
const rolledNote = spool.rolledChars > 0 ? `[!] ${spool.rolledChars} earlier char(s) dropped from the spool (memory bound)\n` : "";
|
|
317
|
+
const rolledNote = spool.rolledChars > 0 ? `[!] ${spool.rolledChars} earlier char(s) dropped from the spool (memory bound)${monitorSpillNote(handle)}\n` : "";
|
|
220
318
|
const droppedNote = (spool.droppedBytes ?? 0) > 0
|
|
221
319
|
? `[!] ${spool.droppedBytes} earlier byte(s) permanently dropped before this window (8MB buffer evicted)\n`
|
|
222
320
|
: spool.dropUnknown === true
|
|
@@ -244,6 +342,28 @@ export async function pollMonitorLane(handle, filter, deadline, signal) {
|
|
|
244
342
|
},
|
|
245
343
|
};
|
|
246
344
|
}
|
|
345
|
+
function consumePendingSuppressionNote(handle) {
|
|
346
|
+
const suppressed = handle.suppressedBatches ?? 0;
|
|
347
|
+
if (suppressed <= 0)
|
|
348
|
+
return "";
|
|
349
|
+
handle.suppressedBatches = 0;
|
|
350
|
+
return ` [${suppressed} event batch(es) suppressed before the watch ended — output rate was too high; the suppressed output is still readable via TaskOutput.]`;
|
|
351
|
+
}
|
|
352
|
+
function explicitStopTerminalNote(handle, alreadyGone) {
|
|
353
|
+
const label = (handle.description ?? "monitor").slice(0, 200);
|
|
354
|
+
const detail = alreadyGone
|
|
355
|
+
? "killed before completion (stopped via TaskStop; the process was already gone); watch ended."
|
|
356
|
+
: "killed before completion (stopped via TaskStop); watch ended.";
|
|
357
|
+
emitMonitorEventLane(handle, {
|
|
358
|
+
task_id: handle.id,
|
|
359
|
+
task_type: "monitor",
|
|
360
|
+
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
361
|
+
status: "killed",
|
|
362
|
+
stoppedBy: handle.stoppedBy,
|
|
363
|
+
summary: `${terminalTaskSummary("monitor", label, "killed")} — ${detail}${droppedGapNote(handle.spool)}`,
|
|
364
|
+
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
247
367
|
export async function stopMonitorLane(core, handle) {
|
|
248
368
|
const wasRunning = handle.status === "running";
|
|
249
369
|
if (handle.watcher !== undefined) {
|
|
@@ -266,9 +386,9 @@ export async function stopMonitorLane(core, handle) {
|
|
|
266
386
|
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
|
|
267
387
|
status: "event",
|
|
268
388
|
seq: handle.seq,
|
|
269
|
-
lines: finalLines,
|
|
389
|
+
lines: finalLines.map(clipMonitorLine),
|
|
270
390
|
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${finalLines.length} final output line(s) drained at stop${firstDropNote(handle.spool)}`,
|
|
271
|
-
result: clipMonitorEvent(finalLines.join("\n")),
|
|
391
|
+
result: clipMonitorEvent(finalLines.map(clipMonitorLine).join("\n")),
|
|
272
392
|
});
|
|
273
393
|
}
|
|
274
394
|
}
|
|
@@ -285,7 +405,7 @@ export async function stopMonitorLane(core, handle) {
|
|
|
285
405
|
const r = await handle.env.killBackground(handle.shellId);
|
|
286
406
|
if (handle.status !== "running") {
|
|
287
407
|
return {
|
|
288
|
-
content: wasRunning ? `Terminated ${handle.id}.${drainNote}` : alreadyTerminalStopNote(handle.id, handle.status),
|
|
408
|
+
content: wasRunning ? `Terminated ${handle.id}.${drainNote}${consumePendingSuppressionNote(handle)}` : alreadyTerminalStopNote(handle.id, handle.status),
|
|
289
409
|
details: {
|
|
290
410
|
task_id: handle.id,
|
|
291
411
|
type: "monitor",
|
|
@@ -303,9 +423,9 @@ export async function stopMonitorLane(core, handle) {
|
|
|
303
423
|
handle.status = "killed";
|
|
304
424
|
handle.updatedAt = Date.now();
|
|
305
425
|
mintCompletionId(handle);
|
|
306
|
-
core.
|
|
426
|
+
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, true));
|
|
307
427
|
return {
|
|
308
|
-
content: `Terminated ${handle.id} (the process was already gone).${drainNote}`,
|
|
428
|
+
content: `Terminated ${handle.id} (the process was already gone).${drainNote}${consumePendingSuppressionNote(handle)}`,
|
|
309
429
|
details: {
|
|
310
430
|
task_id: handle.id,
|
|
311
431
|
type: "monitor",
|
|
@@ -331,10 +451,10 @@ export async function stopMonitorLane(core, handle) {
|
|
|
331
451
|
handle.status = "killed";
|
|
332
452
|
handle.updatedAt = Date.now();
|
|
333
453
|
mintCompletionId(handle);
|
|
334
|
-
core.
|
|
454
|
+
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, false));
|
|
335
455
|
}
|
|
336
456
|
return {
|
|
337
|
-
content: `Terminated ${handle.id}.${drainNote}`,
|
|
457
|
+
content: `Terminated ${handle.id}.${drainNote}${consumePendingSuppressionNote(handle)}`,
|
|
338
458
|
details: {
|
|
339
459
|
task_id: handle.id,
|
|
340
460
|
type: "monitor",
|
|
@@ -4,6 +4,7 @@ import { type WorkflowRunStore } from "./workflow-run-store.js";
|
|
|
4
4
|
import type { BackgroundAgentRecord, BackgroundAgentStore } from "./background-agent-store.js";
|
|
5
5
|
import type { WorkflowHandle, WorkflowRun } from "../orchestration/workflow.js";
|
|
6
6
|
import type { TaskNotificationPayload } from "./task-notification.js";
|
|
7
|
+
import type { ToolResultStore } from "./tool-result-store.js";
|
|
7
8
|
export type SemaTaskType = "background_bash" | "workflow" | "background_agent" | "monitor";
|
|
8
9
|
export type StopSource = "user" | "parent" | "system" | (string & {});
|
|
9
10
|
export type SemaTaskStatus = "pending" | "running" | "parked" | "completed" | "failed" | "killed" | "cancelled";
|
|
@@ -132,6 +133,7 @@ export interface BackgroundAgentTaskHandle extends SemaTaskHandle {
|
|
|
132
133
|
sessionScoped?: true;
|
|
133
134
|
result?: string;
|
|
134
135
|
resultFull?: string;
|
|
136
|
+
spillRef?: string;
|
|
135
137
|
error?: string;
|
|
136
138
|
resultIsPartial?: boolean;
|
|
137
139
|
stopSource?: StopSource;
|
|
@@ -161,7 +163,13 @@ export interface MonitorTaskHandle extends SemaTaskHandle {
|
|
|
161
163
|
};
|
|
162
164
|
lineBuf: string;
|
|
163
165
|
seq: number;
|
|
164
|
-
|
|
166
|
+
stormTokens: number;
|
|
167
|
+
stormBurst: number;
|
|
168
|
+
stormRefillIntervalMs: number;
|
|
169
|
+
stormRefilledAt: number;
|
|
170
|
+
suppressedBatches?: number;
|
|
171
|
+
overloadSince?: number;
|
|
172
|
+
lastSuppressedAt?: number;
|
|
165
173
|
batchWindowMs: number;
|
|
166
174
|
maxBatchesPerMinute: number;
|
|
167
175
|
persistent?: true;
|
|
@@ -176,6 +184,13 @@ export interface MonitorTaskHandle extends SemaTaskHandle {
|
|
|
176
184
|
stopSource?: StopSource;
|
|
177
185
|
stoppedBy?: StopSource;
|
|
178
186
|
terminalNotified?: true;
|
|
187
|
+
toolResultStore?: ToolResultStore;
|
|
188
|
+
spillSessionId?: string;
|
|
189
|
+
spillSegCount?: number;
|
|
190
|
+
spillErrSegCount?: number;
|
|
191
|
+
spillBytesUsed?: number;
|
|
192
|
+
spillCapped?: true;
|
|
193
|
+
spillFailed?: true;
|
|
179
194
|
}
|
|
180
195
|
export type RegisteredTaskHandle = BackgroundBashTaskHandle | WorkflowTaskHandle | BackgroundAgentTaskHandle | MonitorTaskHandle;
|
|
181
196
|
export interface RegisterMonitorInput extends TaskAccess {
|
|
@@ -190,7 +205,9 @@ export interface RegisterMonitorInput extends TaskAccess {
|
|
|
190
205
|
timers?: MonitorTimers;
|
|
191
206
|
batchWindowMs?: number;
|
|
192
207
|
maxBatchesPerMinute?: number;
|
|
208
|
+
stormBurst?: number;
|
|
193
209
|
now?: number;
|
|
210
|
+
toolResultStore?: ToolResultStore;
|
|
194
211
|
}
|
|
195
212
|
export interface RegisterWorkflowInput extends TaskAccess {
|
|
196
213
|
runId: string;
|
|
@@ -208,13 +225,19 @@ export declare const MONITOR_BATCH_WINDOW_MS = 200;
|
|
|
208
225
|
export declare const MONITOR_DEFAULT_TIMEOUT_MS = 300000;
|
|
209
226
|
export declare const MONITOR_MAX_TIMEOUT_MS = 3600000;
|
|
210
227
|
export declare const MONITOR_MAX_BATCHES_PER_MINUTE = 50;
|
|
228
|
+
export declare const MONITOR_STORM_BURST = 10;
|
|
229
|
+
export declare const MONITOR_STORM_KILL_AFTER_MS = 30000;
|
|
230
|
+
export declare const MONITOR_LINE_BUF_CAP = 1048576;
|
|
211
231
|
export declare function clipMonitorEvent(s: string): string;
|
|
232
|
+
export declare function clipMonitorLine(s: string): string;
|
|
233
|
+
export declare function terminalTaskSummary(kind: "bash" | "monitor", label: string, status: "completed" | "failed" | "killed", exitCode?: number): string;
|
|
212
234
|
export declare const TASK_OUTPUT_MAX_CHARS = 160000;
|
|
213
235
|
export declare function clipTaskOutput(s: string, fullOutputPath?: string): string;
|
|
214
236
|
export declare function statusFromBackground(status: string, exitCode?: number): SemaTaskStatus;
|
|
215
237
|
export declare function rollSpoolText(spool: {
|
|
216
238
|
rolledChars: number;
|
|
217
|
-
}, s: string, cap: number): string;
|
|
239
|
+
}, s: string, cap: number, onDrop?: (dropped: string) => void): string;
|
|
240
|
+
export declare const MONITOR_SPILL_CAP_BYTES: number;
|
|
218
241
|
export declare function accountDroppedBytes(spool: {
|
|
219
242
|
droppedBytes?: number;
|
|
220
243
|
dropUnknown?: true;
|
|
@@ -32,10 +32,31 @@ export const MONITOR_BATCH_WINDOW_MS = 200;
|
|
|
32
32
|
export const MONITOR_DEFAULT_TIMEOUT_MS = 300_000;
|
|
33
33
|
export const MONITOR_MAX_TIMEOUT_MS = 3_600_000;
|
|
34
34
|
export const MONITOR_MAX_BATCHES_PER_MINUTE = 50;
|
|
35
|
+
export const MONITOR_STORM_BURST = 10;
|
|
36
|
+
export const MONITOR_STORM_KILL_AFTER_MS = 30_000;
|
|
35
37
|
const MONITOR_EVENT_RESULT_CAP = 2_000;
|
|
38
|
+
const MONITOR_EVENT_LINE_CAP = 500;
|
|
39
|
+
export const MONITOR_LINE_BUF_CAP = 1_048_576;
|
|
36
40
|
export function clipMonitorEvent(s) {
|
|
37
41
|
return s.length <= MONITOR_EVENT_RESULT_CAP ? s : `${s.slice(0, MONITOR_EVENT_RESULT_CAP)}\n…[+${s.length - MONITOR_EVENT_RESULT_CAP} chars truncated]`;
|
|
38
42
|
}
|
|
43
|
+
export function clipMonitorLine(s) {
|
|
44
|
+
return s.length <= MONITOR_EVENT_LINE_CAP ? s : `${s.slice(0, MONITOR_EVENT_LINE_CAP)}...(truncated)`;
|
|
45
|
+
}
|
|
46
|
+
export function terminalTaskSummary(kind, label, status, exitCode) {
|
|
47
|
+
if (kind === "monitor") {
|
|
48
|
+
if (status === "completed")
|
|
49
|
+
return `Monitor "${label}" stream ended`;
|
|
50
|
+
if (status === "failed")
|
|
51
|
+
return `Monitor "${label}" script failed${exitCode !== undefined ? ` (exit ${exitCode})` : ""}`;
|
|
52
|
+
return `Monitor "${label}" stopped`;
|
|
53
|
+
}
|
|
54
|
+
if (status === "completed")
|
|
55
|
+
return `Background command "${label}" completed${exitCode !== undefined ? ` (exit code ${exitCode})` : ""}`;
|
|
56
|
+
if (status === "failed")
|
|
57
|
+
return `Background command "${label}" failed${exitCode !== undefined ? ` with exit code ${exitCode}` : ""}`;
|
|
58
|
+
return `Background command "${label}" was stopped`;
|
|
59
|
+
}
|
|
39
60
|
const TASK_OUTPUT_DEFAULT_CHARS = 32_000;
|
|
40
61
|
export const TASK_OUTPUT_MAX_CHARS = 160_000;
|
|
41
62
|
const TASK_OUTPUT_MIN_CHARS = 512;
|
|
@@ -64,13 +85,16 @@ export function statusFromBackground(status, exitCode) {
|
|
|
64
85
|
return exitCode === 0 ? "completed" : "failed";
|
|
65
86
|
return "failed";
|
|
66
87
|
}
|
|
67
|
-
export function rollSpoolText(spool, s, cap) {
|
|
88
|
+
export function rollSpoolText(spool, s, cap, onDrop) {
|
|
68
89
|
if (s.length <= cap)
|
|
69
90
|
return s;
|
|
70
91
|
const half = Math.floor(cap / 2);
|
|
71
|
-
|
|
92
|
+
const dropped = s.slice(half, s.length - half);
|
|
93
|
+
spool.rolledChars += dropped.length;
|
|
94
|
+
onDrop?.(dropped);
|
|
72
95
|
return s.slice(0, half) + s.slice(s.length - half);
|
|
73
96
|
}
|
|
97
|
+
export const MONITOR_SPILL_CAP_BYTES = 64 * 1024 * 1024;
|
|
74
98
|
export function accountDroppedBytes(spool, poll) {
|
|
75
99
|
const d = poll.bytesDroppedBeforeCursor ?? 0;
|
|
76
100
|
if (d > 0)
|
|
@@ -4,6 +4,7 @@ import { type BackgroundShellCapability, type BackgroundShellId } from "./backgr
|
|
|
4
4
|
import type { WorkflowRunStore } from "./workflow-run-store.js";
|
|
5
5
|
import { type BackgroundAgentStore } from "./background-agent-store.js";
|
|
6
6
|
import type { TaskNotificationPayload } from "./task-notification.js";
|
|
7
|
+
import type { ToolResultStore } from "./tool-result-store.js";
|
|
7
8
|
import { type SemaTaskType, type StopSource, type SemaTaskStatus, type TaskRetrievalStatus, type UnifiedTaskOutput, type UnifiedTaskResult, type SemaTaskHandle, type TaskAccess, type BackgroundAgentTaskHandle, type MonitorTimers, type RegisterMonitorInput, type RegisterWorkflowInput, canAccessWorkflowRun, clipTaskOutput, MONITOR_BATCH_WINDOW_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_MAX_TIMEOUT_MS, MONITOR_MAX_BATCHES_PER_MINUTE } from "./task-registry-shared.js";
|
|
8
9
|
export { normalizeAgentName, DURABLE_AGENT_HEARTBEAT_MS, DURABLE_AGENT_HANDLE_RE, BG_AGENT_REAP_STOP_ERROR } from "./task-registry-shared.js";
|
|
9
10
|
export type { ParkedClaimTicket, RegisterBackgroundAgentInput } from "./task-registry-shared.js";
|
|
@@ -32,6 +33,8 @@ export interface TaskPollOptions {
|
|
|
32
33
|
block?: boolean;
|
|
33
34
|
timeoutMs?: number;
|
|
34
35
|
signal?: AbortSignal;
|
|
36
|
+
oneShot?: boolean;
|
|
37
|
+
toolResultStore?: ToolResultStore;
|
|
35
38
|
}
|
|
36
39
|
export interface TaskStopOptions {
|
|
37
40
|
workflowStore?: WorkflowRunStore;
|
|
@@ -43,6 +46,8 @@ export interface TaskToolOptions extends TaskAccess {
|
|
|
43
46
|
agentStore?: BackgroundAgentStore;
|
|
44
47
|
deadlineMs?: () => number | undefined;
|
|
45
48
|
notificationWired?: boolean;
|
|
49
|
+
oneShot?: boolean;
|
|
50
|
+
toolResultStore?: ToolResultStore;
|
|
46
51
|
}
|
|
47
52
|
export interface AccessibleTaskRow {
|
|
48
53
|
task_id: string;
|