@poncho-ai/harness 0.37.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -5
- package/CHANGELOG.md +216 -0
- package/dist/index.d.ts +397 -34
- package/dist/index.js +2228 -130
- package/package.json +2 -2
- package/src/harness.ts +48 -72
- package/src/index.ts +1 -0
- package/src/local-tools.ts +4 -3
- package/src/mcp.ts +9 -6
- package/src/orchestrator/continuation.ts +13 -0
- package/src/orchestrator/history.ts +69 -0
- package/src/orchestrator/index.ts +54 -0
- package/src/orchestrator/orchestrator.ts +1535 -0
- package/src/orchestrator/subagents.ts +27 -0
- package/src/orchestrator/turn.ts +367 -0
- package/src/reminder-store.ts +183 -16
- package/src/reminder-tools.ts +102 -6
- package/src/state.ts +127 -3
- package/src/storage/engine.ts +29 -10
- package/src/storage/memory-engine.ts +74 -10
- package/src/storage/postgres-engine.ts +1 -0
- package/src/storage/schema.ts +21 -0
- package/src/storage/sql-dialect.ts +294 -23
- package/src/storage/sqlite-engine.ts +1 -0
- package/src/storage/store-adapters.ts +17 -11
- package/src/telemetry.ts +10 -7
- package/src/upload-store.ts +7 -4
- package/src/vfs/bash-manager.ts +16 -2
- package/src/vfs/edit-file-tool.ts +9 -8
- package/src/vfs/read-file-tool.ts +14 -15
- package/src/vfs/write-file-tool.ts +6 -5
- package/test/orchestrator.test.ts +176 -0
- package/test/reminder-store.test.ts +193 -4
- package/test/state.test.ts +21 -0
- package/test/storage-engine.test.ts +80 -0
|
@@ -0,0 +1,1535 @@
|
|
|
1
|
+
import type { AgentEvent, Message } from "@poncho-ai/sdk";
|
|
2
|
+
import type { Conversation, ConversationStore, PendingSubagentResult } from "../state.js";
|
|
3
|
+
import type { AgentHarness } from "../harness.js";
|
|
4
|
+
import type { TelemetryEmitter } from "../telemetry.js";
|
|
5
|
+
import type { SubagentManager, SubagentSpawnResult } from "../subagent-manager.js";
|
|
6
|
+
import {
|
|
7
|
+
executeConversationTurn,
|
|
8
|
+
flushTurnDraft,
|
|
9
|
+
buildAssistantMetadata,
|
|
10
|
+
applyTurnMetadata,
|
|
11
|
+
normalizeApprovalCheckpoint,
|
|
12
|
+
buildApprovalCheckpoints,
|
|
13
|
+
createTurnDraftState,
|
|
14
|
+
recordStandardTurnEvent,
|
|
15
|
+
type TurnDraftState,
|
|
16
|
+
type ExecuteTurnResult,
|
|
17
|
+
type StoredApproval,
|
|
18
|
+
} from "./turn.js";
|
|
19
|
+
import { withToolResultArchiveParam, MAX_CONTINUATION_COUNT } from "./continuation.js";
|
|
20
|
+
import { resolveRunRequest } from "./history.js";
|
|
21
|
+
import {
|
|
22
|
+
type ActiveSubagentRun,
|
|
23
|
+
type PendingSubagentApproval,
|
|
24
|
+
MAX_SUBAGENT_NESTING,
|
|
25
|
+
MAX_CONCURRENT_SUBAGENTS,
|
|
26
|
+
MAX_SUBAGENT_CALLBACK_COUNT,
|
|
27
|
+
CALLBACK_LOCK_STALE_MS,
|
|
28
|
+
STALE_SUBAGENT_THRESHOLD_MS,
|
|
29
|
+
} from "./subagents.js";
|
|
30
|
+
|
|
31
|
+
// ── Types ──
|
|
32
|
+
|
|
33
|
+
export type ActiveConversationRun = {
|
|
34
|
+
ownerId: string;
|
|
35
|
+
abortController: AbortController;
|
|
36
|
+
runId: string | null;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type EventSink = (conversationId: string, event: AgentEvent) => void | Promise<void>;
|
|
40
|
+
|
|
41
|
+
export interface OrchestratorHooks {
|
|
42
|
+
/** Called before a continuation run starts. Resets/creates event streams. */
|
|
43
|
+
onContinuationStart?(conversationId: string): void;
|
|
44
|
+
/** Called after a continuation run finishes. Cleans up event streams. */
|
|
45
|
+
onContinuationEnd?(conversationId: string): void;
|
|
46
|
+
/** Called when an approval checkpoint is stored during resumeRunFromCheckpoint.
|
|
47
|
+
* Transport layer can use this for platform-specific notifications (e.g. Telegram). */
|
|
48
|
+
onApprovalCheckpoint?(conversationId: string, approvals: Array<{ approvalId: string; tool: string; input: Record<string, unknown> }>): void;
|
|
49
|
+
/** Called after resumeRunFromCheckpoint completes. Transport layer can check
|
|
50
|
+
* for pending subagent callbacks and manage stream lifecycle.
|
|
51
|
+
* @deprecated Orchestrator handles post-resume subagent work internally in Phase 5+. */
|
|
52
|
+
onResumeComplete?(conversationId: string, checkpointedRun: boolean): void;
|
|
53
|
+
|
|
54
|
+
// ── Subagent hooks ──
|
|
55
|
+
|
|
56
|
+
/** Create a child AgentHarness for subagent execution. Required for subagent support. */
|
|
57
|
+
createChildHarness?(): Promise<AgentHarness>;
|
|
58
|
+
/** Build recall parameters injected into run parameters for conversation context. */
|
|
59
|
+
buildRecallParams?(opts: { ownerId: string; tenantId?: string | null; excludeConversationId: string }): Record<string, unknown>;
|
|
60
|
+
/** Dispatch a background task via serverless self-fetch. If not provided,
|
|
61
|
+
* the orchestrator calls methods directly (long-lived mode). */
|
|
62
|
+
dispatchBackground?(type: "subagent-run" | "subagent-callback" | "continuation", conversationId: string): void;
|
|
63
|
+
/** Called when a conversation's event stream should be closed/finished. */
|
|
64
|
+
onStreamEnd?(conversationId: string): void;
|
|
65
|
+
/** Called when processSubagentCallback needs to open/reset the parent's event stream. */
|
|
66
|
+
onCallbackStreamReset?(conversationId: string): void;
|
|
67
|
+
/** Notify a messaging platform about a subagent callback result. */
|
|
68
|
+
onMessagingNotify?(conversationId: string, text: string): void;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** @deprecated Use OrchestratorHooks instead */
|
|
72
|
+
export type ContinuationHooks = OrchestratorHooks;
|
|
73
|
+
|
|
74
|
+
export interface OrchestratorOptions {
|
|
75
|
+
harness: AgentHarness;
|
|
76
|
+
conversationStore: ConversationStore;
|
|
77
|
+
eventSink: EventSink;
|
|
78
|
+
telemetry?: TelemetryEmitter;
|
|
79
|
+
hooks?: OrchestratorHooks;
|
|
80
|
+
/** @deprecated Use hooks instead */
|
|
81
|
+
continuationHooks?: OrchestratorHooks;
|
|
82
|
+
/** Agent identity ID for tool context. Falls back to harness frontmatter ID. */
|
|
83
|
+
agentId?: string;
|
|
84
|
+
/** Working directory for tool context. */
|
|
85
|
+
workingDir?: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ── AgentOrchestrator ──
|
|
89
|
+
|
|
90
|
+
export class AgentOrchestrator {
|
|
91
|
+
readonly harness: AgentHarness;
|
|
92
|
+
readonly conversationStore: ConversationStore;
|
|
93
|
+
readonly eventSink: EventSink;
|
|
94
|
+
readonly telemetry?: TelemetryEmitter;
|
|
95
|
+
/** @internal */
|
|
96
|
+
hooks?: OrchestratorHooks;
|
|
97
|
+
private readonly agentId: string;
|
|
98
|
+
private readonly workingDir: string;
|
|
99
|
+
|
|
100
|
+
// ── Runtime state (conversation runs) ──
|
|
101
|
+
readonly activeConversationRuns = new Map<string, ActiveConversationRun>();
|
|
102
|
+
readonly runOwners = new Map<string, string>();
|
|
103
|
+
readonly runConversations = new Map<string, string>();
|
|
104
|
+
readonly approvalDecisionTracker = new Map<string, Map<string, boolean>>();
|
|
105
|
+
|
|
106
|
+
// ── Runtime state (subagents) ──
|
|
107
|
+
readonly activeSubagentRuns = new Map<string, ActiveSubagentRun>();
|
|
108
|
+
readonly recentlySpawnedParents = new Map<string, number>();
|
|
109
|
+
readonly pendingSubagentApprovals = new Map<string, PendingSubagentApproval>();
|
|
110
|
+
readonly pendingCallbackNeeded = new Set<string>();
|
|
111
|
+
|
|
112
|
+
constructor(options: OrchestratorOptions) {
|
|
113
|
+
this.harness = options.harness;
|
|
114
|
+
this.conversationStore = options.conversationStore;
|
|
115
|
+
this.eventSink = options.eventSink;
|
|
116
|
+
this.telemetry = options.telemetry;
|
|
117
|
+
this.hooks = options.hooks ?? options.continuationHooks;
|
|
118
|
+
this.agentId = options.agentId ?? options.harness.frontmatter?.id ?? "";
|
|
119
|
+
this.workingDir = options.workingDir ?? "";
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
setHooks(hooks: OrchestratorHooks): void {
|
|
123
|
+
this.hooks = hooks;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** @deprecated Use setHooks instead */
|
|
127
|
+
setContinuationHooks(hooks: OrchestratorHooks): void {
|
|
128
|
+
this.hooks = hooks;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private get isServerless(): boolean {
|
|
132
|
+
return typeof this.hooks?.dispatchBackground === "function";
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
136
|
+
// Continuation
|
|
137
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
138
|
+
|
|
139
|
+
async runContinuation(
|
|
140
|
+
conversationId: string,
|
|
141
|
+
onYield?: (event: AgentEvent) => void | Promise<void>,
|
|
142
|
+
): Promise<void> {
|
|
143
|
+
const conversation = await this.conversationStore.getWithArchive(conversationId);
|
|
144
|
+
if (!conversation) return;
|
|
145
|
+
if (Array.isArray(conversation.pendingApprovals) && conversation.pendingApprovals.length > 0) return;
|
|
146
|
+
if (!conversation._continuationMessages?.length) return;
|
|
147
|
+
if (conversation.runStatus === "running") return;
|
|
148
|
+
|
|
149
|
+
const count = (conversation._continuationCount ?? 0) + 1;
|
|
150
|
+
if (count > MAX_CONTINUATION_COUNT) {
|
|
151
|
+
console.warn(`[poncho][continuation] Max continuation count (${MAX_CONTINUATION_COUNT}) reached for ${conversationId}`);
|
|
152
|
+
conversation._continuationMessages = undefined;
|
|
153
|
+
conversation._continuationCount = undefined;
|
|
154
|
+
await this.conversationStore.update(conversation);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const continuationMessages = [...conversation._continuationMessages];
|
|
159
|
+
conversation._continuationMessages = undefined;
|
|
160
|
+
conversation._continuationCount = count;
|
|
161
|
+
conversation.runStatus = "running";
|
|
162
|
+
await this.conversationStore.update(conversation);
|
|
163
|
+
|
|
164
|
+
const abortController = new AbortController();
|
|
165
|
+
this.activeConversationRuns.set(conversationId, {
|
|
166
|
+
ownerId: conversation.ownerId,
|
|
167
|
+
abortController,
|
|
168
|
+
runId: null,
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
this.hooks?.onContinuationStart?.(conversationId);
|
|
172
|
+
|
|
173
|
+
try {
|
|
174
|
+
if (conversation.parentConversationId) {
|
|
175
|
+
for await (const event of this.runSubagentContinuation(conversationId, conversation, continuationMessages)) {
|
|
176
|
+
if (onYield) await onYield(event);
|
|
177
|
+
}
|
|
178
|
+
} else {
|
|
179
|
+
await this.runChatContinuation(conversationId, conversation, continuationMessages, onYield);
|
|
180
|
+
}
|
|
181
|
+
} finally {
|
|
182
|
+
this.activeConversationRuns.delete(conversationId);
|
|
183
|
+
this.hooks?.onContinuationEnd?.(conversationId);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async runChatContinuation(
|
|
188
|
+
conversationId: string,
|
|
189
|
+
conversation: Conversation,
|
|
190
|
+
continuationMessages: Message[],
|
|
191
|
+
onYield?: (event: AgentEvent) => void | Promise<void>,
|
|
192
|
+
): Promise<void> {
|
|
193
|
+
const execution = await executeConversationTurn({
|
|
194
|
+
harness: this.harness,
|
|
195
|
+
runInput: {
|
|
196
|
+
conversationId,
|
|
197
|
+
tenantId: conversation.tenantId ?? undefined,
|
|
198
|
+
parameters: withToolResultArchiveParam({
|
|
199
|
+
__activeConversationId: conversationId,
|
|
200
|
+
__ownerId: conversation.ownerId,
|
|
201
|
+
}, conversation),
|
|
202
|
+
messages: continuationMessages,
|
|
203
|
+
abortSignal: this.activeConversationRuns.get(conversationId)?.abortController.signal,
|
|
204
|
+
},
|
|
205
|
+
initialContextTokens: conversation.contextTokens ?? 0,
|
|
206
|
+
initialContextWindow: conversation.contextWindow ?? 0,
|
|
207
|
+
onEvent: async (event) => {
|
|
208
|
+
if (event.type === "run:started") {
|
|
209
|
+
this.runOwners.set(event.runId, conversation.ownerId);
|
|
210
|
+
this.runConversations.set(event.runId, conversationId);
|
|
211
|
+
const active = this.activeConversationRuns.get(conversationId);
|
|
212
|
+
if (active) active.runId = event.runId;
|
|
213
|
+
}
|
|
214
|
+
if (this.telemetry) await this.telemetry.emit(event);
|
|
215
|
+
await this.eventSink(conversationId, event);
|
|
216
|
+
if (onYield) await onYield(event);
|
|
217
|
+
},
|
|
218
|
+
});
|
|
219
|
+
flushTurnDraft(execution.draft);
|
|
220
|
+
|
|
221
|
+
const freshConv = await this.conversationStore.get(conversationId);
|
|
222
|
+
if (!freshConv) return;
|
|
223
|
+
|
|
224
|
+
const hasContent = execution.draft.assistantResponse.length > 0 || execution.draft.toolTimeline.length > 0;
|
|
225
|
+
if (hasContent) {
|
|
226
|
+
freshConv.messages = [
|
|
227
|
+
...freshConv.messages,
|
|
228
|
+
{
|
|
229
|
+
role: "assistant" as const,
|
|
230
|
+
content: execution.draft.assistantResponse,
|
|
231
|
+
metadata: buildAssistantMetadata(execution.draft),
|
|
232
|
+
},
|
|
233
|
+
];
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
applyTurnMetadata(freshConv, {
|
|
237
|
+
latestRunId: execution.latestRunId,
|
|
238
|
+
contextTokens: execution.runContextTokens,
|
|
239
|
+
contextWindow: execution.runContextWindow,
|
|
240
|
+
continuation: execution.runContinuation,
|
|
241
|
+
continuationMessages: execution.runContinuationMessages,
|
|
242
|
+
harnessMessages: execution.runHarnessMessages,
|
|
243
|
+
toolResultArchive: this.harness.getToolResultArchive(conversationId),
|
|
244
|
+
}, { shouldRebuildCanonical: true });
|
|
245
|
+
if (execution.runContinuation) {
|
|
246
|
+
freshConv._continuationCount = conversation._continuationCount;
|
|
247
|
+
}
|
|
248
|
+
await this.conversationStore.update(freshConv);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
252
|
+
// Approval checkpoint management
|
|
253
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
254
|
+
|
|
255
|
+
async findPendingApproval(
|
|
256
|
+
approvalId: string,
|
|
257
|
+
owner: string,
|
|
258
|
+
): Promise<{ conversation: Conversation; approval: StoredApproval } | undefined> {
|
|
259
|
+
const searchedConversationIds = new Set<string>();
|
|
260
|
+
const scan = async (conversations: Conversation[]) => {
|
|
261
|
+
for (const conv of conversations) {
|
|
262
|
+
if (searchedConversationIds.has(conv.conversationId)) continue;
|
|
263
|
+
searchedConversationIds.add(conv.conversationId);
|
|
264
|
+
if (!Array.isArray(conv.pendingApprovals)) continue;
|
|
265
|
+
const match = conv.pendingApprovals.find((a) => a.approvalId === approvalId);
|
|
266
|
+
if (match) {
|
|
267
|
+
return { conversation: conv, approval: match as StoredApproval };
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return undefined;
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
const ownerScoped = await scan(await this.conversationStore.list(owner));
|
|
274
|
+
if (ownerScoped) return ownerScoped;
|
|
275
|
+
|
|
276
|
+
if (owner === "local-owner") {
|
|
277
|
+
return await scan(await this.conversationStore.list());
|
|
278
|
+
}
|
|
279
|
+
return undefined;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async resumeRunFromCheckpoint(
|
|
283
|
+
conversationId: string,
|
|
284
|
+
conversation: Conversation,
|
|
285
|
+
checkpoint: StoredApproval,
|
|
286
|
+
toolResults: Array<{ callId: string; toolName: string; result?: unknown; error?: string }>,
|
|
287
|
+
): Promise<void> {
|
|
288
|
+
const abortController = new AbortController();
|
|
289
|
+
this.activeConversationRuns.set(conversationId, {
|
|
290
|
+
ownerId: conversation.ownerId,
|
|
291
|
+
abortController,
|
|
292
|
+
runId: null,
|
|
293
|
+
});
|
|
294
|
+
let latestRunId = conversation.runtimeRunId ?? "";
|
|
295
|
+
let checkpointedRun = false;
|
|
296
|
+
|
|
297
|
+
const normalizedCheckpoint = normalizeApprovalCheckpoint(checkpoint, conversation.messages);
|
|
298
|
+
const baseMessages = normalizedCheckpoint.baseMessageCount != null
|
|
299
|
+
? conversation.messages.slice(0, normalizedCheckpoint.baseMessageCount)
|
|
300
|
+
: [];
|
|
301
|
+
const fullCheckpointMessages = [...baseMessages, ...normalizedCheckpoint.checkpointMessages!];
|
|
302
|
+
|
|
303
|
+
let resumeToolResultMsg: Message | undefined;
|
|
304
|
+
const lastCpMsg = fullCheckpointMessages[fullCheckpointMessages.length - 1];
|
|
305
|
+
if (lastCpMsg?.role === "assistant") {
|
|
306
|
+
try {
|
|
307
|
+
const parsed = JSON.parse(typeof lastCpMsg.content === "string" ? lastCpMsg.content : "");
|
|
308
|
+
const cpToolCalls: Array<{ id: string; name: string }> = parsed.tool_calls ?? [];
|
|
309
|
+
if (cpToolCalls.length > 0) {
|
|
310
|
+
const providedMap = new Map(toolResults.map(r => [r.callId, r]));
|
|
311
|
+
resumeToolResultMsg = {
|
|
312
|
+
role: "tool",
|
|
313
|
+
content: JSON.stringify(cpToolCalls.map(tc => {
|
|
314
|
+
const provided = providedMap.get(tc.id);
|
|
315
|
+
return {
|
|
316
|
+
type: "tool_result",
|
|
317
|
+
tool_use_id: tc.id,
|
|
318
|
+
tool_name: provided?.toolName ?? tc.name,
|
|
319
|
+
content: provided
|
|
320
|
+
? (provided.error ? `Tool error: ${provided.error}` : JSON.stringify(provided.result ?? null))
|
|
321
|
+
: "Tool error: Tool execution deferred (pending approval checkpoint)",
|
|
322
|
+
};
|
|
323
|
+
})),
|
|
324
|
+
metadata: { timestamp: Date.now() },
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
} catch { /* last message is not a parseable assistant-with-tools -- skip */ }
|
|
328
|
+
}
|
|
329
|
+
const fullCheckpointWithResults = resumeToolResultMsg
|
|
330
|
+
? [...fullCheckpointMessages, resumeToolResultMsg]
|
|
331
|
+
: fullCheckpointMessages;
|
|
332
|
+
|
|
333
|
+
let draftRef: TurnDraftState | undefined;
|
|
334
|
+
let execution: ExecuteTurnResult | undefined;
|
|
335
|
+
|
|
336
|
+
try {
|
|
337
|
+
execution = await executeConversationTurn({
|
|
338
|
+
harness: this.harness,
|
|
339
|
+
events: this.harness.continueFromToolResult({
|
|
340
|
+
messages: fullCheckpointMessages,
|
|
341
|
+
toolResults,
|
|
342
|
+
conversationId,
|
|
343
|
+
abortSignal: abortController.signal,
|
|
344
|
+
}),
|
|
345
|
+
initialContextTokens: conversation.contextTokens ?? 0,
|
|
346
|
+
initialContextWindow: conversation.contextWindow ?? 0,
|
|
347
|
+
onEvent: async (event, draft) => {
|
|
348
|
+
draftRef = draft;
|
|
349
|
+
if (event.type === "run:started") {
|
|
350
|
+
latestRunId = event.runId;
|
|
351
|
+
this.runOwners.set(event.runId, conversation.ownerId);
|
|
352
|
+
this.runConversations.set(event.runId, conversationId);
|
|
353
|
+
const active = this.activeConversationRuns.get(conversationId);
|
|
354
|
+
if (active && active.abortController === abortController) {
|
|
355
|
+
active.runId = event.runId;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if (event.type === "tool:approval:required") {
|
|
359
|
+
const toolText = `- approval required \`${(event as { tool: string }).tool}\``;
|
|
360
|
+
draft.toolTimeline.push(toolText);
|
|
361
|
+
draft.currentTools.push(toolText);
|
|
362
|
+
}
|
|
363
|
+
if (event.type === "tool:approval:checkpoint") {
|
|
364
|
+
const cpEvent = event as {
|
|
365
|
+
approvals: Array<{ approvalId: string; tool: string; toolCallId: string; input: Record<string, unknown> }>;
|
|
366
|
+
checkpointMessages: Message[];
|
|
367
|
+
pendingToolCalls: Array<{ id: string; name: string; input: Record<string, unknown> }>;
|
|
368
|
+
};
|
|
369
|
+
const conv = await this.conversationStore.get(conversationId);
|
|
370
|
+
if (conv) {
|
|
371
|
+
conv.pendingApprovals = buildApprovalCheckpoints({
|
|
372
|
+
approvals: cpEvent.approvals,
|
|
373
|
+
runId: latestRunId,
|
|
374
|
+
checkpointMessages: [...fullCheckpointWithResults, ...cpEvent.checkpointMessages],
|
|
375
|
+
baseMessageCount: 0,
|
|
376
|
+
pendingToolCalls: cpEvent.pendingToolCalls,
|
|
377
|
+
});
|
|
378
|
+
conv.updatedAt = Date.now();
|
|
379
|
+
await this.conversationStore.update(conv);
|
|
380
|
+
this.hooks?.onApprovalCheckpoint?.(
|
|
381
|
+
conversationId,
|
|
382
|
+
cpEvent.approvals.map(a => ({ approvalId: a.approvalId, tool: a.tool, input: a.input })),
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
checkpointedRun = true;
|
|
386
|
+
}
|
|
387
|
+
if (this.telemetry) await this.telemetry.emit(event);
|
|
388
|
+
await this.eventSink(conversationId, event);
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
|
+
flushTurnDraft(execution.draft);
|
|
392
|
+
latestRunId = execution.latestRunId || latestRunId;
|
|
393
|
+
} catch (err) {
|
|
394
|
+
console.error("[resume-run] error:", err instanceof Error ? err.message : err);
|
|
395
|
+
if (draftRef) {
|
|
396
|
+
draftRef.assistantResponse = draftRef.assistantResponse || `[Error: ${err instanceof Error ? err.message : "Unknown error"}]`;
|
|
397
|
+
flushTurnDraft(draftRef);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const draft = execution?.draft ?? draftRef ?? createTurnDraftState();
|
|
402
|
+
|
|
403
|
+
if (!checkpointedRun) {
|
|
404
|
+
const conv = await this.conversationStore.get(conversationId);
|
|
405
|
+
if (conv) {
|
|
406
|
+
const hasAssistantContent =
|
|
407
|
+
draft.assistantResponse.length > 0 || draft.toolTimeline.length > 0 || draft.sections.length > 0;
|
|
408
|
+
if (hasAssistantContent) {
|
|
409
|
+
const prevMessages = conv.messages;
|
|
410
|
+
const lastMsg = prevMessages[prevMessages.length - 1];
|
|
411
|
+
if (lastMsg && lastMsg.role === "assistant" && lastMsg.metadata) {
|
|
412
|
+
const existingToolActivity = (lastMsg.metadata as Record<string, unknown>).toolActivity;
|
|
413
|
+
const existingSections = (lastMsg.metadata as Record<string, unknown>).sections;
|
|
414
|
+
const mergedTimeline = [
|
|
415
|
+
...(Array.isArray(existingToolActivity) ? existingToolActivity as string[] : []),
|
|
416
|
+
...draft.toolTimeline,
|
|
417
|
+
];
|
|
418
|
+
const mergedSections = [
|
|
419
|
+
...(Array.isArray(existingSections) ? existingSections as Array<{ type: "text" | "tools"; content: string | string[] }> : []),
|
|
420
|
+
...draft.sections,
|
|
421
|
+
];
|
|
422
|
+
const mergedText = (typeof lastMsg.content === "string" ? lastMsg.content : "") + draft.assistantResponse;
|
|
423
|
+
conv.messages = [
|
|
424
|
+
...prevMessages.slice(0, -1),
|
|
425
|
+
{
|
|
426
|
+
role: "assistant" as const,
|
|
427
|
+
content: mergedText,
|
|
428
|
+
metadata: {
|
|
429
|
+
toolActivity: mergedTimeline,
|
|
430
|
+
sections: mergedSections.length > 0 ? mergedSections : undefined,
|
|
431
|
+
} as Message["metadata"],
|
|
432
|
+
},
|
|
433
|
+
];
|
|
434
|
+
} else {
|
|
435
|
+
conv.messages = [
|
|
436
|
+
...prevMessages,
|
|
437
|
+
{
|
|
438
|
+
role: "assistant" as const,
|
|
439
|
+
content: draft.assistantResponse,
|
|
440
|
+
metadata: buildAssistantMetadata(draft),
|
|
441
|
+
},
|
|
442
|
+
];
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
applyTurnMetadata(conv, {
|
|
446
|
+
latestRunId,
|
|
447
|
+
contextTokens: execution?.runContextTokens ?? 0,
|
|
448
|
+
contextWindow: execution?.runContextWindow ?? 0,
|
|
449
|
+
harnessMessages: execution?.runHarnessMessages,
|
|
450
|
+
}, { shouldRebuildCanonical: true });
|
|
451
|
+
await this.conversationStore.update(conv);
|
|
452
|
+
}
|
|
453
|
+
} else {
|
|
454
|
+
const conv = await this.conversationStore.get(conversationId);
|
|
455
|
+
if (conv) {
|
|
456
|
+
conv.runStatus = "idle";
|
|
457
|
+
conv.updatedAt = Date.now();
|
|
458
|
+
await this.conversationStore.update(conv);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
this.activeConversationRuns.delete(conversationId);
|
|
463
|
+
if (latestRunId) {
|
|
464
|
+
this.runOwners.delete(latestRunId);
|
|
465
|
+
this.runConversations.delete(latestRunId);
|
|
466
|
+
}
|
|
467
|
+
console.log("[resume-run] complete for", conversationId);
|
|
468
|
+
|
|
469
|
+
// Post-resume: check for pending subagent work
|
|
470
|
+
if (this.hooks?.onResumeComplete) {
|
|
471
|
+
// Legacy hook path (if consumer still provides onResumeComplete)
|
|
472
|
+
this.hooks.onResumeComplete(conversationId, checkpointedRun);
|
|
473
|
+
} else {
|
|
474
|
+
await this._handlePostResumeWork(conversationId);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/** After a resume completes, check for deferred subagent callbacks. */
|
|
479
|
+
private async _handlePostResumeWork(conversationId: string): Promise<void> {
|
|
480
|
+
const hadDeferred = this.pendingCallbackNeeded.delete(conversationId);
|
|
481
|
+
const postConv = await this.conversationStore.get(conversationId);
|
|
482
|
+
const needsCallback = hadDeferred || !!postConv?.pendingSubagentResults?.length;
|
|
483
|
+
const hasRunningChildren = this.hasRunningSubagentsForParent(conversationId);
|
|
484
|
+
|
|
485
|
+
if (!needsCallback && !hasRunningChildren) {
|
|
486
|
+
this.hooks?.onStreamEnd?.(conversationId);
|
|
487
|
+
}
|
|
488
|
+
if (needsCallback) {
|
|
489
|
+
this.processSubagentCallback(conversationId, true).catch(err =>
|
|
490
|
+
console.error(`[poncho][subagent-callback] Post-resume callback failed:`, err instanceof Error ? err.message : err),
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
496
|
+
// Subagent lifecycle
|
|
497
|
+
// ══════════════════════════════════════════════════════════════════════════
|
|
498
|
+
|
|
499
|
+
// ── Helpers ──
|
|
500
|
+
|
|
501
|
+
hasRunningSubagentsForParent(parentConversationId: string): boolean {
|
|
502
|
+
for (const run of this.activeSubagentRuns.values()) {
|
|
503
|
+
if (run.parentConversationId === parentConversationId) return true;
|
|
504
|
+
}
|
|
505
|
+
return false;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
getRunningSubagentCountForParent(parentId: string): number {
|
|
509
|
+
let count = 0;
|
|
510
|
+
for (const run of this.activeSubagentRuns.values()) {
|
|
511
|
+
if (run.parentConversationId === parentId) count += 1;
|
|
512
|
+
}
|
|
513
|
+
return count;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
async getSubagentDepth(conversationId: string): Promise<number> {
|
|
517
|
+
let depth = 0;
|
|
518
|
+
let current = await this.conversationStore.get(conversationId);
|
|
519
|
+
while (current?.parentConversationId) {
|
|
520
|
+
depth += 1;
|
|
521
|
+
current = await this.conversationStore.get(current.parentConversationId);
|
|
522
|
+
}
|
|
523
|
+
return depth;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
async hasPendingSubagentWorkForParent(
|
|
527
|
+
parentConversationId: string,
|
|
528
|
+
_owner: string,
|
|
529
|
+
): Promise<boolean> {
|
|
530
|
+
if (this.hasRunningSubagentsForParent(parentConversationId)) return true;
|
|
531
|
+
if (this.recentlySpawnedParents.has(parentConversationId)) return true;
|
|
532
|
+
if (this.pendingCallbackNeeded.has(parentConversationId)) return true;
|
|
533
|
+
const parentConversation = await this.conversationStore.get(parentConversationId);
|
|
534
|
+
if (!parentConversation) return false;
|
|
535
|
+
if (Array.isArray(parentConversation.pendingSubagentResults) && parentConversation.pendingSubagentResults.length > 0) return true;
|
|
536
|
+
if (typeof parentConversation.runningCallbackSince === "number" && parentConversation.runningCallbackSince > 0) return true;
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// ── Subagent approval decision ──
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Submit an approval decision for a pending subagent approval.
|
|
544
|
+
* Returns whether the approval was found, the child conversation, and whether all approvals are now decided.
|
|
545
|
+
*/
|
|
546
|
+
async submitSubagentApprovalDecision(
|
|
547
|
+
approvalId: string,
|
|
548
|
+
approved: boolean,
|
|
549
|
+
): Promise<{ found: boolean; childConversationId?: string; allDecided: boolean }> {
|
|
550
|
+
const pending = this.pendingSubagentApprovals.get(approvalId);
|
|
551
|
+
if (!pending) return { found: false, allDecided: false };
|
|
552
|
+
|
|
553
|
+
const decision = approved ? "approved" as const : "denied" as const;
|
|
554
|
+
pending.checkpoint.decision = decision;
|
|
555
|
+
|
|
556
|
+
await this.eventSink(pending.childConversationId,
|
|
557
|
+
approved
|
|
558
|
+
? { type: "tool:approval:granted", approvalId }
|
|
559
|
+
: { type: "tool:approval:denied", approvalId },
|
|
560
|
+
);
|
|
561
|
+
|
|
562
|
+
// Explicitly update the decision in the conversation store (the store may
|
|
563
|
+
// serialize/deserialize, so in-memory mutation of pending.checkpoint alone
|
|
564
|
+
// is not sufficient).
|
|
565
|
+
const childConv = await this.conversationStore.get(pending.childConversationId);
|
|
566
|
+
if (childConv && Array.isArray(childConv.pendingApprovals)) {
|
|
567
|
+
childConv.pendingApprovals = childConv.pendingApprovals.map(pa =>
|
|
568
|
+
pa.approvalId === approvalId ? { ...pa, decision } : pa,
|
|
569
|
+
);
|
|
570
|
+
await this.conversationStore.update(childConv);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
const allApprovals = childConv?.pendingApprovals ?? [];
|
|
574
|
+
const allDecided = allApprovals.length > 0 && allApprovals.every(pa => pa.decision != null);
|
|
575
|
+
|
|
576
|
+
if (allDecided) {
|
|
577
|
+
for (const pa of allApprovals) this.pendingSubagentApprovals.delete(pa.approvalId);
|
|
578
|
+
if (childConv) {
|
|
579
|
+
childConv.pendingApprovals = [];
|
|
580
|
+
await this.conversationStore.update(childConv);
|
|
581
|
+
}
|
|
582
|
+
pending.resolve(allApprovals);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
return { found: true, childConversationId: pending.childConversationId, allDecided };
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// ── Completion + checkpoint resume ──
|
|
589
|
+
|
|
590
|
+
async handleSubagentCompletion(subagentId: string): Promise<void> {
|
|
591
|
+
const conv = await this.conversationStore.get(subagentId);
|
|
592
|
+
if (!conv || !conv.parentConversationId) return;
|
|
593
|
+
if (conv.subagentMeta?.status === "completed" || conv.subagentMeta?.status === "error") return;
|
|
594
|
+
|
|
595
|
+
conv.subagentMeta = { ...conv.subagentMeta!, status: "completed" };
|
|
596
|
+
conv.updatedAt = Date.now();
|
|
597
|
+
await this.conversationStore.update(conv);
|
|
598
|
+
|
|
599
|
+
const lastMsg = conv.messages[conv.messages.length - 1];
|
|
600
|
+
const responseText = lastMsg?.role === "assistant" && typeof lastMsg.content === "string" ? lastMsg.content : "";
|
|
601
|
+
const pendingResult: PendingSubagentResult = {
|
|
602
|
+
subagentId,
|
|
603
|
+
task: conv.subagentMeta?.task ?? conv.title,
|
|
604
|
+
status: "completed",
|
|
605
|
+
result: { status: "completed", response: responseText, steps: 0, tokens: { input: 0, output: 0, cached: 0 }, duration: 0 },
|
|
606
|
+
timestamp: Date.now(),
|
|
607
|
+
};
|
|
608
|
+
await this.conversationStore.appendSubagentResult(conv.parentConversationId, pendingResult);
|
|
609
|
+
|
|
610
|
+
await this.eventSink(conv.parentConversationId, {
|
|
611
|
+
type: "subagent:completed",
|
|
612
|
+
subagentId,
|
|
613
|
+
conversationId: subagentId,
|
|
614
|
+
});
|
|
615
|
+
|
|
616
|
+
await this.triggerParentCallback(conv.parentConversationId);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
async resumeSubagentFromCheckpoint(subagentId: string): Promise<void> {
|
|
620
|
+
const conv = await this.conversationStore.get(subagentId);
|
|
621
|
+
if (!conv || !conv.parentConversationId) return;
|
|
622
|
+
|
|
623
|
+
const allApprovals = (conv.pendingApprovals ?? []).map((approval) =>
|
|
624
|
+
normalizeApprovalCheckpoint(approval, conv.messages),
|
|
625
|
+
);
|
|
626
|
+
if (allApprovals.length === 0) return;
|
|
627
|
+
const allDecided = allApprovals.every(a => a.decision != null);
|
|
628
|
+
if (!allDecided) return;
|
|
629
|
+
|
|
630
|
+
conv.pendingApprovals = [];
|
|
631
|
+
conv.subagentMeta = { ...conv.subagentMeta!, status: "running" };
|
|
632
|
+
await this.conversationStore.update(conv);
|
|
633
|
+
|
|
634
|
+
const checkpointRef = allApprovals[0]!;
|
|
635
|
+
const toolContext = {
|
|
636
|
+
runId: checkpointRef.runId,
|
|
637
|
+
agentId: this.agentId,
|
|
638
|
+
step: 0,
|
|
639
|
+
workingDir: this.workingDir,
|
|
640
|
+
parameters: {},
|
|
641
|
+
conversationId: subagentId,
|
|
642
|
+
};
|
|
643
|
+
|
|
644
|
+
const approvalToolCallIds = new Set(allApprovals.map(a => a.toolCallId));
|
|
645
|
+
const callsToExecute: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
|
|
646
|
+
const deniedResults: Array<{ callId: string; toolName: string; error: string }> = [];
|
|
647
|
+
|
|
648
|
+
for (const a of allApprovals) {
|
|
649
|
+
if (a.decision === "approved" && a.toolCallId) {
|
|
650
|
+
callsToExecute.push({ id: a.toolCallId, name: a.tool, input: a.input });
|
|
651
|
+
} else if (a.decision === "denied" && a.toolCallId) {
|
|
652
|
+
deniedResults.push({ callId: a.toolCallId, toolName: a.tool, error: "Tool execution denied by user" });
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const pendingToolCalls = checkpointRef.pendingToolCalls ?? [];
|
|
657
|
+
for (const tc of pendingToolCalls) {
|
|
658
|
+
if (!approvalToolCallIds.has(tc.id)) callsToExecute.push(tc);
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
let toolResults: Array<{ callId: string; toolName: string; result?: unknown; error?: string }> = [...deniedResults];
|
|
662
|
+
if (callsToExecute.length > 0) {
|
|
663
|
+
const execResults = await this.harness.executeTools(callsToExecute, toolContext);
|
|
664
|
+
toolResults.push(...execResults.map(r => ({
|
|
665
|
+
callId: r.callId,
|
|
666
|
+
toolName: r.tool,
|
|
667
|
+
result: r.output,
|
|
668
|
+
error: r.error,
|
|
669
|
+
})));
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
await this.resumeRunFromCheckpoint(subagentId, conv, checkpointRef, toolResults);
|
|
673
|
+
await this.handleSubagentCompletion(subagentId);
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// ── Subagent run ──
|
|
677
|
+
|
|
678
|
+
async runSubagent(
|
|
679
|
+
childConversationId: string,
|
|
680
|
+
parentConversationId: string,
|
|
681
|
+
task: string,
|
|
682
|
+
ownerId: string,
|
|
683
|
+
): Promise<void> {
|
|
684
|
+
if (!this.hooks?.createChildHarness) {
|
|
685
|
+
throw new Error("createChildHarness hook is required for subagent support");
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
const childHarness = await this.hooks.createChildHarness();
|
|
689
|
+
|
|
690
|
+
const childAbortController = new AbortController();
|
|
691
|
+
this.activeSubagentRuns.set(childConversationId, { abortController: childAbortController, harness: childHarness, parentConversationId });
|
|
692
|
+
this.activeConversationRuns.set(childConversationId, {
|
|
693
|
+
ownerId,
|
|
694
|
+
abortController: childAbortController,
|
|
695
|
+
runId: null,
|
|
696
|
+
});
|
|
697
|
+
// Decrement the temporary spawn counter now that we're registered
|
|
698
|
+
const spawnCount = this.recentlySpawnedParents.get(parentConversationId) ?? 0;
|
|
699
|
+
if (spawnCount <= 1) {
|
|
700
|
+
this.recentlySpawnedParents.delete(parentConversationId);
|
|
701
|
+
} else {
|
|
702
|
+
this.recentlySpawnedParents.set(parentConversationId, spawnCount - 1);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
childHarness.unregisterTools(["memory_main_write", "memory_main_edit"]);
|
|
706
|
+
|
|
707
|
+
const draft = createTurnDraftState();
|
|
708
|
+
let latestRunId = "";
|
|
709
|
+
let runResult: { status: "completed" | "error" | "cancelled"; response?: string; steps: number; duration: number; continuation?: boolean; continuationMessages?: Message[] } | undefined;
|
|
710
|
+
|
|
711
|
+
try {
|
|
712
|
+
const conversation = await this.conversationStore.getWithArchive(childConversationId);
|
|
713
|
+
if (!conversation) throw new Error("Subagent conversation not found");
|
|
714
|
+
|
|
715
|
+
if (conversation.subagentMeta?.status === "stopped") return;
|
|
716
|
+
|
|
717
|
+
conversation.lastActivityAt = Date.now();
|
|
718
|
+
await this.conversationStore.update(conversation);
|
|
719
|
+
|
|
720
|
+
const runOutcome = resolveRunRequest(conversation, {
|
|
721
|
+
conversationId: childConversationId,
|
|
722
|
+
messages: conversation.messages,
|
|
723
|
+
});
|
|
724
|
+
const harnessMessages = [...runOutcome.messages];
|
|
725
|
+
|
|
726
|
+
const recallParams = this.hooks?.buildRecallParams?.({ ownerId, tenantId: conversation.tenantId, excludeConversationId: childConversationId }) ?? {};
|
|
727
|
+
|
|
728
|
+
for await (const event of childHarness.runWithTelemetry({
|
|
729
|
+
task,
|
|
730
|
+
conversationId: childConversationId,
|
|
731
|
+
tenantId: conversation.tenantId ?? undefined,
|
|
732
|
+
parameters: withToolResultArchiveParam({
|
|
733
|
+
...recallParams,
|
|
734
|
+
__activeConversationId: childConversationId,
|
|
735
|
+
__ownerId: ownerId,
|
|
736
|
+
}, conversation),
|
|
737
|
+
messages: harnessMessages,
|
|
738
|
+
abortSignal: childAbortController.signal,
|
|
739
|
+
})) {
|
|
740
|
+
if (event.type === "run:started") {
|
|
741
|
+
latestRunId = event.runId;
|
|
742
|
+
const active = this.activeConversationRuns.get(childConversationId);
|
|
743
|
+
if (active) active.runId = event.runId;
|
|
744
|
+
}
|
|
745
|
+
recordStandardTurnEvent(draft, event);
|
|
746
|
+
if (event.type === "tool:approval:required") {
|
|
747
|
+
const toolText = `- approval required \`${event.tool}\``;
|
|
748
|
+
draft.toolTimeline.push(toolText);
|
|
749
|
+
draft.currentTools.push(toolText);
|
|
750
|
+
await this.eventSink(parentConversationId, {
|
|
751
|
+
type: "subagent:approval_needed",
|
|
752
|
+
subagentId: childConversationId,
|
|
753
|
+
conversationId: childConversationId,
|
|
754
|
+
tool: event.tool,
|
|
755
|
+
approvalId: event.approvalId,
|
|
756
|
+
input: event.input as Record<string, unknown> | undefined,
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
if (event.type === "tool:approval:checkpoint") {
|
|
760
|
+
const cpConv = await this.conversationStore.get(childConversationId);
|
|
761
|
+
if (cpConv) {
|
|
762
|
+
const allCpData = buildApprovalCheckpoints({
|
|
763
|
+
approvals: event.approvals,
|
|
764
|
+
runId: latestRunId,
|
|
765
|
+
checkpointMessages: [...harnessMessages, ...event.checkpointMessages],
|
|
766
|
+
baseMessageCount: 0,
|
|
767
|
+
pendingToolCalls: event.pendingToolCalls,
|
|
768
|
+
});
|
|
769
|
+
cpConv.pendingApprovals = allCpData;
|
|
770
|
+
cpConv.updatedAt = Date.now();
|
|
771
|
+
await this.conversationStore.update(cpConv);
|
|
772
|
+
|
|
773
|
+
const decidedApprovals = await new Promise<NonNullable<Conversation["pendingApprovals"]>>((resolve) => {
|
|
774
|
+
for (const cpData of allCpData) {
|
|
775
|
+
this.pendingSubagentApprovals.set(cpData.approvalId, {
|
|
776
|
+
resolve,
|
|
777
|
+
childHarness,
|
|
778
|
+
checkpoint: cpData,
|
|
779
|
+
childConversationId,
|
|
780
|
+
parentConversationId,
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
const checkpointRef = normalizeApprovalCheckpoint(allCpData[0]!, [...harnessMessages]);
|
|
786
|
+
const toolContext = {
|
|
787
|
+
runId: checkpointRef.runId,
|
|
788
|
+
agentId: this.agentId,
|
|
789
|
+
step: 0,
|
|
790
|
+
workingDir: this.workingDir,
|
|
791
|
+
parameters: {},
|
|
792
|
+
conversationId: childConversationId,
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
const approvalToolCallIds = new Set(decidedApprovals.map(a => a.toolCallId));
|
|
796
|
+
const callsToExecute: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
|
|
797
|
+
const deniedResults: Array<{ callId: string; toolName: string; error: string }> = [];
|
|
798
|
+
|
|
799
|
+
for (const a of decidedApprovals) {
|
|
800
|
+
if (a.decision === "approved" && a.toolCallId) {
|
|
801
|
+
callsToExecute.push({ id: a.toolCallId, name: a.tool, input: a.input });
|
|
802
|
+
const toolText = `- done \`${a.tool}\``;
|
|
803
|
+
draft.toolTimeline.push(toolText);
|
|
804
|
+
draft.currentTools.push(toolText);
|
|
805
|
+
} else if (a.toolCallId) {
|
|
806
|
+
deniedResults.push({ callId: a.toolCallId, toolName: a.tool, error: "Tool execution denied by user" });
|
|
807
|
+
const toolText = `- denied \`${a.tool}\``;
|
|
808
|
+
draft.toolTimeline.push(toolText);
|
|
809
|
+
draft.currentTools.push(toolText);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
const cpPendingToolCalls = checkpointRef.pendingToolCalls ?? [];
|
|
814
|
+
for (const tc of cpPendingToolCalls) {
|
|
815
|
+
if (!approvalToolCallIds.has(tc.id)) {
|
|
816
|
+
callsToExecute.push(tc);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
let toolResults: Array<{ callId: string; toolName: string; result?: unknown; error?: string }> = [...deniedResults];
|
|
821
|
+
if (callsToExecute.length > 0) {
|
|
822
|
+
const execResults = await childHarness.executeTools(callsToExecute, toolContext);
|
|
823
|
+
toolResults.push(...execResults.map(r => ({
|
|
824
|
+
callId: r.callId,
|
|
825
|
+
toolName: r.tool,
|
|
826
|
+
result: r.output,
|
|
827
|
+
error: r.error,
|
|
828
|
+
})));
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
const resumeMessages = [...checkpointRef.checkpointMessages!];
|
|
832
|
+
for await (const resumeEvent of childHarness.continueFromToolResult({
|
|
833
|
+
messages: resumeMessages,
|
|
834
|
+
toolResults,
|
|
835
|
+
conversationId: childConversationId,
|
|
836
|
+
abortSignal: childAbortController.signal,
|
|
837
|
+
})) {
|
|
838
|
+
recordStandardTurnEvent(draft, resumeEvent);
|
|
839
|
+
if (resumeEvent.type === "run:completed") {
|
|
840
|
+
runResult = { status: resumeEvent.result.status, response: resumeEvent.result.response, steps: resumeEvent.result.steps, duration: resumeEvent.result.duration };
|
|
841
|
+
if (draft.assistantResponse.length === 0 && resumeEvent.result.response) {
|
|
842
|
+
draft.assistantResponse = resumeEvent.result.response;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
if (resumeEvent.type === "run:error") {
|
|
846
|
+
draft.assistantResponse = draft.assistantResponse || `[Error: ${resumeEvent.error.message}]`;
|
|
847
|
+
}
|
|
848
|
+
await this.eventSink(childConversationId, resumeEvent);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
if (event.type === "run:completed") {
|
|
853
|
+
runResult = { status: event.result.status, response: event.result.response, steps: event.result.steps, duration: event.result.duration, continuation: event.result.continuation, continuationMessages: event.result.continuationMessages };
|
|
854
|
+
if (draft.assistantResponse.length === 0 && event.result.response) {
|
|
855
|
+
draft.assistantResponse = event.result.response;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
if (event.type === "run:error") {
|
|
859
|
+
draft.assistantResponse = draft.assistantResponse || `[Error: ${event.error.message}]`;
|
|
860
|
+
}
|
|
861
|
+
await this.eventSink(childConversationId, event);
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
// Persist assistant turn
|
|
865
|
+
flushTurnDraft(draft);
|
|
866
|
+
|
|
867
|
+
const conv = await this.conversationStore.get(childConversationId);
|
|
868
|
+
if (conv) {
|
|
869
|
+
const hasContent = draft.assistantResponse.length > 0 || draft.toolTimeline.length > 0;
|
|
870
|
+
if (hasContent) {
|
|
871
|
+
conv.messages.push({
|
|
872
|
+
role: "assistant",
|
|
873
|
+
content: draft.assistantResponse,
|
|
874
|
+
metadata: buildAssistantMetadata(draft),
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
if (runResult?.continuation && runResult.continuationMessages) {
|
|
878
|
+
conv._continuationMessages = runResult.continuationMessages;
|
|
879
|
+
} else {
|
|
880
|
+
conv._continuationMessages = undefined;
|
|
881
|
+
conv._continuationCount = undefined;
|
|
882
|
+
}
|
|
883
|
+
if (runResult?.continuationMessages) {
|
|
884
|
+
conv._harnessMessages = runResult.continuationMessages;
|
|
885
|
+
} else if (runOutcome.shouldRebuildCanonical) {
|
|
886
|
+
conv._harnessMessages = conv.messages;
|
|
887
|
+
}
|
|
888
|
+
conv._toolResultArchive = childHarness.getToolResultArchive(childConversationId);
|
|
889
|
+
conv.lastActivityAt = Date.now();
|
|
890
|
+
conv.updatedAt = Date.now();
|
|
891
|
+
|
|
892
|
+
if (runResult?.continuation) {
|
|
893
|
+
await this.conversationStore.update(conv);
|
|
894
|
+
this.hooks?.onStreamEnd?.(childConversationId);
|
|
895
|
+
this.activeSubagentRuns.delete(childConversationId);
|
|
896
|
+
this.activeConversationRuns.delete(childConversationId);
|
|
897
|
+
try { await childHarness.shutdown(); } catch {}
|
|
898
|
+
|
|
899
|
+
if (this.isServerless) {
|
|
900
|
+
this.hooks!.dispatchBackground!("continuation", childConversationId);
|
|
901
|
+
} else {
|
|
902
|
+
this.runContinuation(childConversationId).catch(err =>
|
|
903
|
+
console.error(`[poncho][subagent] Continuation failed:`, err instanceof Error ? err.message : err),
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
conv.subagentMeta = { ...conv.subagentMeta!, status: "completed" };
|
|
910
|
+
await this.conversationStore.update(conv);
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
this.hooks?.onStreamEnd?.(childConversationId);
|
|
914
|
+
await this.eventSink(parentConversationId, {
|
|
915
|
+
type: "subagent:completed",
|
|
916
|
+
subagentId: childConversationId,
|
|
917
|
+
conversationId: childConversationId,
|
|
918
|
+
});
|
|
919
|
+
|
|
920
|
+
let subagentResponse = runResult?.response ?? draft.assistantResponse;
|
|
921
|
+
if (!subagentResponse) {
|
|
922
|
+
const freshSubConv = await this.conversationStore.get(childConversationId);
|
|
923
|
+
if (freshSubConv) {
|
|
924
|
+
const lastAssistant = [...freshSubConv.messages].reverse().find(m => m.role === "assistant");
|
|
925
|
+
if (lastAssistant && typeof lastAssistant.content === "string") {
|
|
926
|
+
subagentResponse = lastAssistant.content;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
const pendingResult: PendingSubagentResult = {
|
|
931
|
+
subagentId: childConversationId,
|
|
932
|
+
task,
|
|
933
|
+
status: "completed",
|
|
934
|
+
result: runResult ? { status: runResult.status, response: subagentResponse, steps: runResult.steps, tokens: { input: 0, output: 0, cached: 0 }, duration: runResult.duration } : undefined,
|
|
935
|
+
timestamp: Date.now(),
|
|
936
|
+
};
|
|
937
|
+
await this.conversationStore.appendSubagentResult(parentConversationId, pendingResult);
|
|
938
|
+
this.triggerParentCallback(parentConversationId).catch(err =>
|
|
939
|
+
console.error(`[poncho][subagent] Parent callback failed:`, err instanceof Error ? err.message : err),
|
|
940
|
+
);
|
|
941
|
+
} catch (err) {
|
|
942
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
943
|
+
console.error(`[poncho][subagent] Error in subagent ${childConversationId}:`, errMsg);
|
|
944
|
+
|
|
945
|
+
const conv = await this.conversationStore.get(childConversationId);
|
|
946
|
+
if (conv) {
|
|
947
|
+
conv.subagentMeta = {
|
|
948
|
+
...conv.subagentMeta!,
|
|
949
|
+
status: "error",
|
|
950
|
+
error: { code: "SUBAGENT_ERROR", message: errMsg },
|
|
951
|
+
};
|
|
952
|
+
conv.updatedAt = Date.now();
|
|
953
|
+
await this.conversationStore.update(conv);
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
this.hooks?.onStreamEnd?.(childConversationId);
|
|
957
|
+
await this.eventSink(parentConversationId, {
|
|
958
|
+
type: "subagent:error",
|
|
959
|
+
subagentId: childConversationId,
|
|
960
|
+
conversationId: childConversationId,
|
|
961
|
+
error: errMsg,
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
const pendingResult: PendingSubagentResult = {
|
|
965
|
+
subagentId: childConversationId,
|
|
966
|
+
task,
|
|
967
|
+
status: "error",
|
|
968
|
+
error: { code: "SUBAGENT_ERROR", message: errMsg },
|
|
969
|
+
timestamp: Date.now(),
|
|
970
|
+
};
|
|
971
|
+
await this.conversationStore.appendSubagentResult(parentConversationId, pendingResult).catch(() => {});
|
|
972
|
+
this.triggerParentCallback(parentConversationId).catch(err2 =>
|
|
973
|
+
console.error(`[poncho][subagent] Parent callback failed:`, err2 instanceof Error ? err2.message : err2),
|
|
974
|
+
);
|
|
975
|
+
} finally {
|
|
976
|
+
for (const [aid, pa] of this.pendingSubagentApprovals) {
|
|
977
|
+
if (pa.childHarness === childHarness) {
|
|
978
|
+
this.pendingSubagentApprovals.delete(aid);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
this.activeSubagentRuns.delete(childConversationId);
|
|
982
|
+
this.activeConversationRuns.delete(childConversationId);
|
|
983
|
+
try { await childHarness.shutdown(); } catch {}
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// ── Parent callback ──
|
|
988
|
+
|
|
989
|
+
async triggerParentCallback(parentConversationId: string): Promise<void> {
|
|
990
|
+
if (this.activeConversationRuns.has(parentConversationId)) {
|
|
991
|
+
this.pendingCallbackNeeded.add(parentConversationId);
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
if (this.isServerless) {
|
|
995
|
+
this.hooks!.dispatchBackground!("subagent-callback", parentConversationId);
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
998
|
+
await this.processSubagentCallback(parentConversationId);
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
async processSubagentCallback(conversationId: string, skipLockCheck = false): Promise<void> {
|
|
1002
|
+
const conversation = await this.conversationStore.getWithArchive(conversationId);
|
|
1003
|
+
if (!conversation) return;
|
|
1004
|
+
|
|
1005
|
+
const pendingResults = conversation.pendingSubagentResults ?? [];
|
|
1006
|
+
const hasOrphanedContinuation = pendingResults.length === 0
|
|
1007
|
+
&& Array.isArray(conversation._continuationMessages)
|
|
1008
|
+
&& conversation._continuationMessages.length > 0
|
|
1009
|
+
&& !this.activeConversationRuns.has(conversationId);
|
|
1010
|
+
if (pendingResults.length === 0 && !hasOrphanedContinuation) return;
|
|
1011
|
+
|
|
1012
|
+
// Store-based lock for serverless
|
|
1013
|
+
if (!skipLockCheck && conversation.runningCallbackSince) {
|
|
1014
|
+
const elapsed = Date.now() - conversation.runningCallbackSince;
|
|
1015
|
+
if (elapsed < CALLBACK_LOCK_STALE_MS) return;
|
|
1016
|
+
console.warn(`[poncho][subagent-callback] Stale lock detected (${elapsed}ms) for ${conversationId}, proceeding`);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
// Acquire lock and clear pending
|
|
1020
|
+
conversation.pendingSubagentResults = [];
|
|
1021
|
+
conversation.runningCallbackSince = Date.now();
|
|
1022
|
+
conversation.runStatus = "running";
|
|
1023
|
+
const callbackCount = (conversation.subagentCallbackCount ?? 0) + 1;
|
|
1024
|
+
conversation.subagentCallbackCount = callbackCount;
|
|
1025
|
+
|
|
1026
|
+
for (const pr of pendingResults) {
|
|
1027
|
+
const resultBody = pr.result
|
|
1028
|
+
? `Status: ${pr.result.status}\nResponse: ${pr.result.response ?? "(no response)"}\nSteps: ${pr.result.steps}, Duration: ${pr.result.duration}ms`
|
|
1029
|
+
: pr.error
|
|
1030
|
+
? `Error: ${pr.error.message}`
|
|
1031
|
+
: "(no result)";
|
|
1032
|
+
conversation.messages.push({
|
|
1033
|
+
role: "user",
|
|
1034
|
+
content: `[Subagent Result] Subagent "${pr.task}" (${pr.subagentId}) ${pr.status}:\n\n${resultBody}`,
|
|
1035
|
+
metadata: { _subagentCallback: true, subagentId: pr.subagentId, task: pr.task, timestamp: pr.timestamp } as Message["metadata"],
|
|
1036
|
+
});
|
|
1037
|
+
}
|
|
1038
|
+
const processedIds = new Set(pendingResults.map(pr => pr.subagentId));
|
|
1039
|
+
const freshForPending = await this.conversationStore.get(conversationId);
|
|
1040
|
+
const arrivedDuringCallback = (freshForPending?.pendingSubagentResults ?? [])
|
|
1041
|
+
.filter(pr => !processedIds.has(pr.subagentId));
|
|
1042
|
+
conversation.pendingSubagentResults = arrivedDuringCallback;
|
|
1043
|
+
conversation._harnessMessages = [...conversation.messages];
|
|
1044
|
+
conversation.updatedAt = Date.now();
|
|
1045
|
+
await this.conversationStore.update(conversation);
|
|
1046
|
+
|
|
1047
|
+
if (callbackCount > MAX_SUBAGENT_CALLBACK_COUNT) {
|
|
1048
|
+
console.warn(`[poncho][subagent-callback] Circuit breaker: ${callbackCount} callbacks for ${conversationId}, skipping re-run`);
|
|
1049
|
+
conversation.runningCallbackSince = undefined;
|
|
1050
|
+
conversation.runStatus = "idle";
|
|
1051
|
+
await this.conversationStore.update(conversation);
|
|
1052
|
+
return;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
const isContinuationResume = hasOrphanedContinuation && pendingResults.length === 0;
|
|
1056
|
+
console.log(`[poncho][subagent-callback] Processing ${pendingResults.length} result(s) for ${conversationId} (callback #${callbackCount})${isContinuationResume ? " (continuation resume)" : ""}`);
|
|
1057
|
+
|
|
1058
|
+
const abortController = new AbortController();
|
|
1059
|
+
this.activeConversationRuns.set(conversationId, {
|
|
1060
|
+
ownerId: conversation.ownerId,
|
|
1061
|
+
abortController,
|
|
1062
|
+
runId: null,
|
|
1063
|
+
});
|
|
1064
|
+
this.hooks?.onCallbackStreamReset?.(conversationId);
|
|
1065
|
+
|
|
1066
|
+
const historySelection = resolveRunRequest(conversation, {
|
|
1067
|
+
conversationId,
|
|
1068
|
+
messages: conversation.messages,
|
|
1069
|
+
preferContinuation: isContinuationResume,
|
|
1070
|
+
});
|
|
1071
|
+
const historyMessages = [...historySelection.messages];
|
|
1072
|
+
console.info(
|
|
1073
|
+
`[poncho][subagent-callback] conversation="${conversationId}" history_source=${historySelection.source}`,
|
|
1074
|
+
);
|
|
1075
|
+
let execution: ExecuteTurnResult | undefined;
|
|
1076
|
+
|
|
1077
|
+
try {
|
|
1078
|
+
execution = await executeConversationTurn({
|
|
1079
|
+
harness: this.harness,
|
|
1080
|
+
runInput: {
|
|
1081
|
+
task: undefined,
|
|
1082
|
+
conversationId,
|
|
1083
|
+
tenantId: conversation.tenantId ?? undefined,
|
|
1084
|
+
parameters: withToolResultArchiveParam({
|
|
1085
|
+
__activeConversationId: conversationId,
|
|
1086
|
+
__ownerId: conversation.ownerId,
|
|
1087
|
+
}, conversation),
|
|
1088
|
+
messages: historyMessages,
|
|
1089
|
+
abortSignal: abortController.signal,
|
|
1090
|
+
},
|
|
1091
|
+
initialContextTokens: conversation.contextTokens ?? 0,
|
|
1092
|
+
initialContextWindow: conversation.contextWindow ?? 0,
|
|
1093
|
+
onEvent: (event) => {
|
|
1094
|
+
if (event.type === "run:started") {
|
|
1095
|
+
const active = this.activeConversationRuns.get(conversationId);
|
|
1096
|
+
if (active) active.runId = event.runId;
|
|
1097
|
+
}
|
|
1098
|
+
this.eventSink(conversationId, event);
|
|
1099
|
+
},
|
|
1100
|
+
});
|
|
1101
|
+
flushTurnDraft(execution.draft);
|
|
1102
|
+
|
|
1103
|
+
const callbackNeedsContinuation = execution.runContinuation && execution.runContinuationMessages;
|
|
1104
|
+
if (callbackNeedsContinuation || execution.draft.assistantResponse.length > 0 || execution.draft.toolTimeline.length > 0) {
|
|
1105
|
+
const freshConv = await this.conversationStore.get(conversationId);
|
|
1106
|
+
if (freshConv) {
|
|
1107
|
+
if (!callbackNeedsContinuation) {
|
|
1108
|
+
freshConv.messages.push({
|
|
1109
|
+
role: "assistant",
|
|
1110
|
+
content: execution.draft.assistantResponse,
|
|
1111
|
+
metadata: buildAssistantMetadata(execution.draft),
|
|
1112
|
+
});
|
|
1113
|
+
}
|
|
1114
|
+
applyTurnMetadata(freshConv, {
|
|
1115
|
+
latestRunId: execution.latestRunId,
|
|
1116
|
+
contextTokens: execution.runContextTokens,
|
|
1117
|
+
contextWindow: execution.runContextWindow,
|
|
1118
|
+
continuation: !!callbackNeedsContinuation,
|
|
1119
|
+
continuationMessages: execution.runContinuationMessages,
|
|
1120
|
+
harnessMessages: callbackNeedsContinuation ? execution.runHarnessMessages : undefined,
|
|
1121
|
+
toolResultArchive: this.harness.getToolResultArchive(conversationId),
|
|
1122
|
+
}, { shouldRebuildCanonical: true, clearApprovals: false });
|
|
1123
|
+
freshConv.runningCallbackSince = undefined;
|
|
1124
|
+
await this.conversationStore.update(freshConv);
|
|
1125
|
+
|
|
1126
|
+
// Proactive messaging notification
|
|
1127
|
+
if (freshConv.channelMeta && execution.draft.assistantResponse.length > 0) {
|
|
1128
|
+
this.hooks?.onMessagingNotify?.(conversationId, execution.draft.assistantResponse);
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// Handle continuation for the callback run itself
|
|
1134
|
+
if (execution.runContinuation) {
|
|
1135
|
+
if (this.isServerless) {
|
|
1136
|
+
this.hooks!.dispatchBackground!("subagent-callback", conversationId);
|
|
1137
|
+
} else {
|
|
1138
|
+
this.processSubagentCallback(conversationId, true).catch(err =>
|
|
1139
|
+
console.error(`[poncho][subagent-callback] Continuation failed:`, err instanceof Error ? err.message : err),
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
} catch (err) {
|
|
1144
|
+
console.error(`[poncho][subagent-callback] Error during parent re-run for ${conversationId}:`, err instanceof Error ? err.message : err);
|
|
1145
|
+
const errConv = await this.conversationStore.get(conversationId);
|
|
1146
|
+
if (errConv) {
|
|
1147
|
+
errConv.runningCallbackSince = undefined;
|
|
1148
|
+
errConv.runStatus = "idle";
|
|
1149
|
+
await this.conversationStore.update(errConv);
|
|
1150
|
+
}
|
|
1151
|
+
} finally {
|
|
1152
|
+
this.activeConversationRuns.delete(conversationId);
|
|
1153
|
+
|
|
1154
|
+
const hadDeferredTrigger = this.pendingCallbackNeeded.delete(conversationId);
|
|
1155
|
+
const freshConv = await this.conversationStore.get(conversationId);
|
|
1156
|
+
const hasPendingInStore = !!freshConv?.pendingSubagentResults?.length;
|
|
1157
|
+
const hasRunningCallbackChildren = this.hasRunningSubagentsForParent(conversationId);
|
|
1158
|
+
|
|
1159
|
+
if (!hadDeferredTrigger && !hasPendingInStore && !hasRunningCallbackChildren) {
|
|
1160
|
+
this.hooks?.onStreamEnd?.(conversationId);
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
if (hadDeferredTrigger || hasPendingInStore) {
|
|
1164
|
+
if (this.isServerless) {
|
|
1165
|
+
this.hooks!.dispatchBackground!("subagent-callback", conversationId);
|
|
1166
|
+
} else {
|
|
1167
|
+
this.processSubagentCallback(conversationId, true).catch(err =>
|
|
1168
|
+
console.error(`[poncho][subagent-callback] Recursive callback failed:`, err instanceof Error ? err.message : err),
|
|
1169
|
+
);
|
|
1170
|
+
}
|
|
1171
|
+
} else if (freshConv?.runningCallbackSince) {
|
|
1172
|
+
const afterClear = await this.conversationStore.clearCallbackLock(conversationId);
|
|
1173
|
+
if (afterClear?.pendingSubagentResults?.length) {
|
|
1174
|
+
if (this.isServerless) {
|
|
1175
|
+
this.hooks!.dispatchBackground!("subagent-callback", conversationId);
|
|
1176
|
+
} else {
|
|
1177
|
+
this.processSubagentCallback(conversationId, true).catch(err =>
|
|
1178
|
+
console.error(`[poncho][subagent-callback] Post-clear callback failed:`, err instanceof Error ? err.message : err),
|
|
1179
|
+
);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
// ── Subagent continuation ──
|
|
1187
|
+
|
|
1188
|
+
async *runSubagentContinuation(
|
|
1189
|
+
conversationId: string,
|
|
1190
|
+
conversation: Conversation,
|
|
1191
|
+
continuationMessages: Message[],
|
|
1192
|
+
): AsyncGenerator<AgentEvent> {
|
|
1193
|
+
if (!this.hooks?.createChildHarness) {
|
|
1194
|
+
throw new Error("createChildHarness hook is required for subagent support");
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
const parentConversationId = conversation.parentConversationId!;
|
|
1198
|
+
const task = conversation.subagentMeta?.task ?? "";
|
|
1199
|
+
const ownerId = conversation.ownerId;
|
|
1200
|
+
|
|
1201
|
+
const childHarness = await this.hooks.createChildHarness();
|
|
1202
|
+
childHarness.unregisterTools(["memory_main_write", "memory_main_edit"]);
|
|
1203
|
+
|
|
1204
|
+
const childAbortController = this.activeConversationRuns.get(conversationId)?.abortController ?? new AbortController();
|
|
1205
|
+
this.activeSubagentRuns.set(conversationId, { abortController: childAbortController, harness: childHarness, parentConversationId });
|
|
1206
|
+
|
|
1207
|
+
const draft = createTurnDraftState();
|
|
1208
|
+
let runResult: { status: string; response?: string; steps: number; duration: number; continuation?: boolean; continuationMessages?: Message[] } | undefined;
|
|
1209
|
+
|
|
1210
|
+
try {
|
|
1211
|
+
const recallParams = this.hooks?.buildRecallParams?.({ ownerId, tenantId: conversation.tenantId, excludeConversationId: conversationId }) ?? {};
|
|
1212
|
+
|
|
1213
|
+
for await (const event of childHarness.runWithTelemetry({
|
|
1214
|
+
conversationId,
|
|
1215
|
+
tenantId: conversation.tenantId ?? undefined,
|
|
1216
|
+
parameters: withToolResultArchiveParam({
|
|
1217
|
+
...recallParams,
|
|
1218
|
+
__activeConversationId: conversationId,
|
|
1219
|
+
__ownerId: ownerId,
|
|
1220
|
+
}, conversation),
|
|
1221
|
+
messages: continuationMessages,
|
|
1222
|
+
abortSignal: childAbortController.signal,
|
|
1223
|
+
})) {
|
|
1224
|
+
if (event.type === "run:started") {
|
|
1225
|
+
const active = this.activeConversationRuns.get(conversationId);
|
|
1226
|
+
if (active) active.runId = event.runId;
|
|
1227
|
+
}
|
|
1228
|
+
recordStandardTurnEvent(draft, event);
|
|
1229
|
+
if (event.type === "run:completed") {
|
|
1230
|
+
runResult = {
|
|
1231
|
+
status: event.result.status,
|
|
1232
|
+
response: event.result.response,
|
|
1233
|
+
steps: event.result.steps,
|
|
1234
|
+
duration: event.result.duration,
|
|
1235
|
+
continuation: event.result.continuation,
|
|
1236
|
+
continuationMessages: event.result.continuationMessages,
|
|
1237
|
+
};
|
|
1238
|
+
if (!draft.assistantResponse && event.result.response) {
|
|
1239
|
+
draft.assistantResponse = event.result.response;
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
if (event.type === "run:error") {
|
|
1243
|
+
draft.assistantResponse = draft.assistantResponse || `[Error: ${event.error.message}]`;
|
|
1244
|
+
}
|
|
1245
|
+
await this.eventSink(conversationId, event);
|
|
1246
|
+
yield event;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
flushTurnDraft(draft);
|
|
1250
|
+
|
|
1251
|
+
const conv = await this.conversationStore.get(conversationId);
|
|
1252
|
+
if (conv) {
|
|
1253
|
+
const hasContent = draft.assistantResponse.length > 0 || draft.toolTimeline.length > 0;
|
|
1254
|
+
if (runResult?.continuation && runResult.continuationMessages) {
|
|
1255
|
+
if (hasContent) {
|
|
1256
|
+
conv.messages.push({
|
|
1257
|
+
role: "assistant",
|
|
1258
|
+
content: draft.assistantResponse,
|
|
1259
|
+
metadata: buildAssistantMetadata(draft),
|
|
1260
|
+
});
|
|
1261
|
+
}
|
|
1262
|
+
conv._continuationMessages = runResult.continuationMessages;
|
|
1263
|
+
conv._continuationCount = conversation._continuationCount;
|
|
1264
|
+
} else {
|
|
1265
|
+
conv._continuationMessages = undefined;
|
|
1266
|
+
conv._continuationCount = undefined;
|
|
1267
|
+
if (hasContent) {
|
|
1268
|
+
conv.messages.push({
|
|
1269
|
+
role: "assistant",
|
|
1270
|
+
content: draft.assistantResponse,
|
|
1271
|
+
metadata: buildAssistantMetadata(draft),
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
if (runResult?.continuationMessages) {
|
|
1276
|
+
conv._harnessMessages = runResult.continuationMessages;
|
|
1277
|
+
} else {
|
|
1278
|
+
conv._harnessMessages = conv.messages;
|
|
1279
|
+
}
|
|
1280
|
+
conv._toolResultArchive = childHarness.getToolResultArchive(conversationId);
|
|
1281
|
+
conv.lastActivityAt = Date.now();
|
|
1282
|
+
conv.runStatus = "idle";
|
|
1283
|
+
conv.updatedAt = Date.now();
|
|
1284
|
+
|
|
1285
|
+
if (runResult?.continuation) {
|
|
1286
|
+
await this.conversationStore.update(conv);
|
|
1287
|
+
this.activeSubagentRuns.delete(conversationId);
|
|
1288
|
+
try { await childHarness.shutdown(); } catch {}
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
conv.subagentMeta = { ...conv.subagentMeta!, status: "completed" };
|
|
1293
|
+
await this.conversationStore.update(conv);
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
this.activeSubagentRuns.delete(conversationId);
|
|
1297
|
+
await this.eventSink(parentConversationId, {
|
|
1298
|
+
type: "subagent:completed",
|
|
1299
|
+
subagentId: conversationId,
|
|
1300
|
+
conversationId,
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1303
|
+
let subagentResponse = runResult?.response ?? draft.assistantResponse;
|
|
1304
|
+
if (!subagentResponse) {
|
|
1305
|
+
const freshSubConv = await this.conversationStore.get(conversationId);
|
|
1306
|
+
if (freshSubConv) {
|
|
1307
|
+
const lastAssistant = [...freshSubConv.messages].reverse().find(m => m.role === "assistant");
|
|
1308
|
+
if (lastAssistant) {
|
|
1309
|
+
subagentResponse = typeof lastAssistant.content === "string" ? lastAssistant.content : "";
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
const parentConv = await this.conversationStore.get(parentConversationId);
|
|
1315
|
+
if (parentConv) {
|
|
1316
|
+
const result: PendingSubagentResult = {
|
|
1317
|
+
subagentId: conversationId,
|
|
1318
|
+
task,
|
|
1319
|
+
status: "completed",
|
|
1320
|
+
result: { status: "completed", response: subagentResponse, steps: runResult?.steps ?? 0, tokens: { input: 0, output: 0, cached: 0 }, duration: runResult?.duration ?? 0 },
|
|
1321
|
+
timestamp: Date.now(),
|
|
1322
|
+
};
|
|
1323
|
+
await this.conversationStore.appendSubagentResult(parentConversationId, result);
|
|
1324
|
+
|
|
1325
|
+
if (this.isServerless) {
|
|
1326
|
+
this.hooks!.dispatchBackground!("subagent-callback", parentConversationId);
|
|
1327
|
+
} else {
|
|
1328
|
+
this.processSubagentCallback(parentConversationId).catch(err =>
|
|
1329
|
+
console.error(`[poncho][subagent] Callback failed:`, err instanceof Error ? err.message : err),
|
|
1330
|
+
);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
try { await childHarness.shutdown(); } catch {}
|
|
1335
|
+
} catch (err) {
|
|
1336
|
+
this.activeSubagentRuns.delete(conversationId);
|
|
1337
|
+
try { await childHarness.shutdown(); } catch {}
|
|
1338
|
+
|
|
1339
|
+
const conv = await this.conversationStore.get(conversationId);
|
|
1340
|
+
if (conv) {
|
|
1341
|
+
conv.subagentMeta = { ...conv.subagentMeta!, status: "error", error: { code: "CONTINUATION_ERROR", message: err instanceof Error ? err.message : String(err) } };
|
|
1342
|
+
conv.runStatus = "idle";
|
|
1343
|
+
conv._continuationMessages = undefined;
|
|
1344
|
+
conv._continuationCount = undefined;
|
|
1345
|
+
conv.updatedAt = Date.now();
|
|
1346
|
+
await this.conversationStore.update(conv);
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
await this.eventSink(conversation.parentConversationId!, {
|
|
1350
|
+
type: "subagent:completed",
|
|
1351
|
+
subagentId: conversationId,
|
|
1352
|
+
conversationId,
|
|
1353
|
+
});
|
|
1354
|
+
|
|
1355
|
+
const parentConv = await this.conversationStore.get(conversation.parentConversationId!);
|
|
1356
|
+
if (parentConv) {
|
|
1357
|
+
const result: PendingSubagentResult = {
|
|
1358
|
+
subagentId: conversationId,
|
|
1359
|
+
task,
|
|
1360
|
+
status: "error",
|
|
1361
|
+
error: { code: "CONTINUATION_ERROR", message: err instanceof Error ? err.message : String(err) },
|
|
1362
|
+
timestamp: Date.now(),
|
|
1363
|
+
};
|
|
1364
|
+
await this.conversationStore.appendSubagentResult(conversation.parentConversationId!, result);
|
|
1365
|
+
if (this.isServerless) {
|
|
1366
|
+
this.hooks!.dispatchBackground!("subagent-callback", conversation.parentConversationId!);
|
|
1367
|
+
} else {
|
|
1368
|
+
this.processSubagentCallback(conversation.parentConversationId!).catch(() => {});
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
// ── SubagentManager factory ──
|
|
1375
|
+
|
|
1376
|
+
createSubagentManager(): SubagentManager {
|
|
1377
|
+
return {
|
|
1378
|
+
spawn: async (opts): Promise<SubagentSpawnResult> => {
|
|
1379
|
+
const depth = await this.getSubagentDepth(opts.parentConversationId);
|
|
1380
|
+
if (depth >= MAX_SUBAGENT_NESTING - 1) {
|
|
1381
|
+
throw new Error(`Maximum subagent nesting (${MAX_SUBAGENT_NESTING} levels) reached. Cannot spawn deeper subagents.`);
|
|
1382
|
+
}
|
|
1383
|
+
if (this.getRunningSubagentCountForParent(opts.parentConversationId) >= MAX_CONCURRENT_SUBAGENTS) {
|
|
1384
|
+
throw new Error(`Maximum concurrent subagents (${MAX_CONCURRENT_SUBAGENTS}) per parent reached. Wait for running subagents to complete or stop some first.`);
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
const conversation = await this.conversationStore.create(
|
|
1388
|
+
opts.ownerId,
|
|
1389
|
+
opts.task.slice(0, 80),
|
|
1390
|
+
opts.tenantId ?? null,
|
|
1391
|
+
{
|
|
1392
|
+
parentConversationId: opts.parentConversationId,
|
|
1393
|
+
subagentMeta: { task: opts.task, status: "running" },
|
|
1394
|
+
messages: [{ role: "user", content: opts.task }],
|
|
1395
|
+
},
|
|
1396
|
+
);
|
|
1397
|
+
|
|
1398
|
+
this.recentlySpawnedParents.set(
|
|
1399
|
+
opts.parentConversationId,
|
|
1400
|
+
(this.recentlySpawnedParents.get(opts.parentConversationId) ?? 0) + 1,
|
|
1401
|
+
);
|
|
1402
|
+
|
|
1403
|
+
await this.eventSink(opts.parentConversationId, {
|
|
1404
|
+
type: "subagent:spawned",
|
|
1405
|
+
subagentId: conversation.conversationId,
|
|
1406
|
+
conversationId: conversation.conversationId,
|
|
1407
|
+
task: opts.task,
|
|
1408
|
+
});
|
|
1409
|
+
|
|
1410
|
+
if (this.isServerless) {
|
|
1411
|
+
this.hooks!.dispatchBackground!("subagent-run", conversation.conversationId);
|
|
1412
|
+
} else {
|
|
1413
|
+
this.runSubagent(
|
|
1414
|
+
conversation.conversationId,
|
|
1415
|
+
opts.parentConversationId,
|
|
1416
|
+
opts.task,
|
|
1417
|
+
opts.ownerId,
|
|
1418
|
+
).catch(err => console.error(`[poncho][subagent] Background spawn failed:`, err instanceof Error ? err.message : err));
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
return { subagentId: conversation.conversationId };
|
|
1422
|
+
},
|
|
1423
|
+
|
|
1424
|
+
sendMessage: async (subagentId, message): Promise<SubagentSpawnResult> => {
|
|
1425
|
+
const conversation = await this.conversationStore.get(subagentId);
|
|
1426
|
+
if (!conversation) {
|
|
1427
|
+
console.error(`[poncho][subagent] sendMessage: conversation "${subagentId}" not found in store`);
|
|
1428
|
+
throw new Error(`Subagent "${subagentId}" not found.`);
|
|
1429
|
+
}
|
|
1430
|
+
if (!conversation.parentConversationId) {
|
|
1431
|
+
throw new Error(`Conversation "${subagentId}" is not a subagent.`);
|
|
1432
|
+
}
|
|
1433
|
+
if (!conversation.subagentMeta) {
|
|
1434
|
+
console.warn(`[poncho][subagent] sendMessage: conversation "${subagentId}" missing subagentMeta, recovering`);
|
|
1435
|
+
conversation.subagentMeta = { task: conversation.title, status: "stopped" };
|
|
1436
|
+
await this.conversationStore.update(conversation);
|
|
1437
|
+
}
|
|
1438
|
+
if (conversation.subagentMeta.status === "running") {
|
|
1439
|
+
throw new Error(`Subagent "${subagentId}" is currently running. Wait for it to complete before sending a new message.`);
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
conversation.messages.push({ role: "user", content: message });
|
|
1443
|
+
conversation.subagentMeta.status = "running";
|
|
1444
|
+
conversation.updatedAt = Date.now();
|
|
1445
|
+
await this.conversationStore.update(conversation);
|
|
1446
|
+
|
|
1447
|
+
if (this.isServerless) {
|
|
1448
|
+
this.hooks!.dispatchBackground!("subagent-run", subagentId);
|
|
1449
|
+
} else {
|
|
1450
|
+
this.runSubagent(
|
|
1451
|
+
subagentId,
|
|
1452
|
+
conversation.parentConversationId,
|
|
1453
|
+
message,
|
|
1454
|
+
conversation.ownerId,
|
|
1455
|
+
).catch(err => console.error(`[poncho][subagent] Background sendMessage failed:`, err instanceof Error ? err.message : err));
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
return { subagentId };
|
|
1459
|
+
},
|
|
1460
|
+
|
|
1461
|
+
stop: async (subagentId) => {
|
|
1462
|
+
const active = this.activeSubagentRuns.get(subagentId);
|
|
1463
|
+
if (active) {
|
|
1464
|
+
active.abortController.abort();
|
|
1465
|
+
}
|
|
1466
|
+
const conversation = await this.conversationStore.get(subagentId);
|
|
1467
|
+
if (conversation?.subagentMeta && conversation.subagentMeta.status === "running") {
|
|
1468
|
+
conversation.subagentMeta.status = "stopped";
|
|
1469
|
+
conversation.updatedAt = Date.now();
|
|
1470
|
+
await this.conversationStore.update(conversation);
|
|
1471
|
+
}
|
|
1472
|
+
},
|
|
1473
|
+
|
|
1474
|
+
list: async (parentConversationId) => {
|
|
1475
|
+
const parentConv = await this.conversationStore.get(parentConversationId);
|
|
1476
|
+
const summaries = await this.conversationStore.listSummaries(parentConv?.ownerId ?? "local-owner");
|
|
1477
|
+
const childSummaries = summaries.filter((s) => s.parentConversationId === parentConversationId);
|
|
1478
|
+
const results: Array<{ subagentId: string; task: string; status: string; messageCount: number }> = [];
|
|
1479
|
+
for (const s of childSummaries) {
|
|
1480
|
+
const c = await this.conversationStore.get(s.conversationId);
|
|
1481
|
+
if (c) {
|
|
1482
|
+
results.push({
|
|
1483
|
+
subagentId: c.conversationId,
|
|
1484
|
+
task: c.subagentMeta?.task ?? c.title,
|
|
1485
|
+
status: c.subagentMeta?.status ?? "stopped",
|
|
1486
|
+
messageCount: c.messages.length,
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
return results;
|
|
1491
|
+
},
|
|
1492
|
+
};
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
// ── Stale subagent recovery ──
|
|
1496
|
+
|
|
1497
|
+
async recoverStaleSubagents(): Promise<void> {
|
|
1498
|
+
const allSummaries = await this.conversationStore.listSummaries();
|
|
1499
|
+
const subagentSummaries = allSummaries.filter((s) => s.parentConversationId);
|
|
1500
|
+
if (subagentSummaries.length === 0) return;
|
|
1501
|
+
const parentsToCallback = new Set<string>();
|
|
1502
|
+
const CONCURRENCY = 10;
|
|
1503
|
+
for (let i = 0; i < subagentSummaries.length; i += CONCURRENCY) {
|
|
1504
|
+
const batch = subagentSummaries.slice(i, i + CONCURRENCY);
|
|
1505
|
+
const convs = await Promise.all(batch.map((s) => this.conversationStore.get(s.conversationId)));
|
|
1506
|
+
for (const conv of convs) {
|
|
1507
|
+
if (conv?.subagentMeta?.status === "running" && conv.parentConversationId) {
|
|
1508
|
+
const lastActivity = conv.lastActivityAt ?? conv.updatedAt;
|
|
1509
|
+
const elapsed = Date.now() - lastActivity;
|
|
1510
|
+
if (elapsed < STALE_SUBAGENT_THRESHOLD_MS) continue;
|
|
1511
|
+
|
|
1512
|
+
conv.subagentMeta.status = "error";
|
|
1513
|
+
conv.subagentMeta.error = { code: "STALE_SUBAGENT", message: `Subagent inactive for ${Math.round(elapsed / 1000)}s (threshold: ${STALE_SUBAGENT_THRESHOLD_MS / 1000}s)` };
|
|
1514
|
+
conv.updatedAt = Date.now();
|
|
1515
|
+
await this.conversationStore.update(conv);
|
|
1516
|
+
|
|
1517
|
+
const pendingResult: PendingSubagentResult = {
|
|
1518
|
+
subagentId: conv.conversationId,
|
|
1519
|
+
task: conv.subagentMeta.task,
|
|
1520
|
+
status: "error",
|
|
1521
|
+
error: conv.subagentMeta.error,
|
|
1522
|
+
timestamp: Date.now(),
|
|
1523
|
+
};
|
|
1524
|
+
await this.conversationStore.appendSubagentResult(conv.parentConversationId, pendingResult);
|
|
1525
|
+
parentsToCallback.add(conv.parentConversationId);
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
for (const parentId of parentsToCallback) {
|
|
1530
|
+
this.processSubagentCallback(parentId).catch(err =>
|
|
1531
|
+
console.error(`[poncho][subagent] Recovery callback failed for ${parentId}:`, err instanceof Error ? err.message : err),
|
|
1532
|
+
);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
}
|