@bermudi/pi-delegate 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -4
- package/concurrency.ts +55 -16
- package/config.ts +584 -50
- package/delegate.ts +11 -3
- package/dispatch.ts +436 -65
- package/extension.ts +16 -4
- package/format.ts +52 -6
- package/host-cache.ts +70 -0
- package/host.ts +169 -811
- package/isolated-workspace.ts +857 -0
- package/lifecycle.ts +699 -520
- package/manual.ts +4 -7
- package/package.json +1 -1
- package/pi-package-source.ts +293 -0
- package/pool.ts +23 -1
- package/provider-extensions.ts +537 -0
- package/quiescence.ts +262 -0
- package/render-branches.ts +30 -0
- package/render-result.ts +8 -5
- package/runner.ts +35 -141
- package/schema.ts +133 -202
- package/settings.ts +202 -84
- package/shared-write-safety.ts +273 -0
- package/task-resolution.ts +121 -32
- package/telemetry.ts +135 -68
- package/ticket-format.ts +331 -0
- package/tickets.ts +101 -258
- package/tools.ts +12 -0
- package/trusted-paths.ts +71 -0
- package/types.ts +83 -16
package/lifecycle.ts
CHANGED
|
@@ -32,6 +32,9 @@ import { createScratchWorkspace, ScratchDeadlineError } from "./workspace.ts";
|
|
|
32
32
|
/** Internal seam for lifecycle-level tests without replacing session ownership. */
|
|
33
33
|
type RunAgentSession = typeof runAgentSession;
|
|
34
34
|
let runAgentSessionForTesting: RunAgentSession = runAgentSession;
|
|
35
|
+
type CreateScratchWorkspace = typeof createScratchWorkspace;
|
|
36
|
+
let createScratchWorkspaceForTesting: CreateScratchWorkspace =
|
|
37
|
+
createScratchWorkspace;
|
|
35
38
|
|
|
36
39
|
export function _setRunAgentSessionForTesting(
|
|
37
40
|
override: RunAgentSession | undefined,
|
|
@@ -39,6 +42,13 @@ export function _setRunAgentSessionForTesting(
|
|
|
39
42
|
runAgentSessionForTesting = override ?? runAgentSession;
|
|
40
43
|
}
|
|
41
44
|
|
|
45
|
+
/** @internal Test-only scratch materialization seam. */
|
|
46
|
+
export function _setCreateScratchWorkspaceForTesting(
|
|
47
|
+
override: CreateScratchWorkspace | undefined,
|
|
48
|
+
): void {
|
|
49
|
+
createScratchWorkspaceForTesting = override ?? createScratchWorkspace;
|
|
50
|
+
}
|
|
51
|
+
|
|
42
52
|
/**
|
|
43
53
|
* Test-only overrides for whole-task retry settings. When set, these bypass
|
|
44
54
|
* the config-driven values so retry integration tests don't sleep real seconds.
|
|
@@ -60,11 +70,15 @@ export function _setWholeTaskRetryForTesting(
|
|
|
60
70
|
testWholeTaskBaseDelayMs = opts?.baseDelayMs;
|
|
61
71
|
}
|
|
62
72
|
|
|
63
|
-
function resolvedWholeTaskMaxRetries(): number {
|
|
64
|
-
return
|
|
73
|
+
function resolvedWholeTaskMaxRetries(env: TaskRunEnv): number {
|
|
74
|
+
return (
|
|
75
|
+
testWholeTaskMaxRetries ?? getWholeTaskMaxRetries(env.config ?? undefined)
|
|
76
|
+
);
|
|
65
77
|
}
|
|
66
|
-
function resolvedWholeTaskBaseDelayMs(): number {
|
|
67
|
-
return
|
|
78
|
+
function resolvedWholeTaskBaseDelayMs(env: TaskRunEnv): number {
|
|
79
|
+
return (
|
|
80
|
+
testWholeTaskBaseDelayMs ?? getWholeTaskBaseDelayMs(env.config ?? undefined)
|
|
81
|
+
);
|
|
68
82
|
}
|
|
69
83
|
|
|
70
84
|
/** Build a failed TaskResult. Used for early-failure paths (abort, busy, validation). */
|
|
@@ -152,6 +166,19 @@ interface RunUpdateOffset {
|
|
|
152
166
|
tokensOffset?: number;
|
|
153
167
|
toolUsesOffset?: number;
|
|
154
168
|
}
|
|
169
|
+
/** Advance cumulative progress counters without ever moving backwards —
|
|
170
|
+
* a live TaskProgress row is monotonic across attempts and retries. */
|
|
171
|
+
function bumpProgressCounters(
|
|
172
|
+
p: TaskProgress,
|
|
173
|
+
tokens: number,
|
|
174
|
+
toolUses: number,
|
|
175
|
+
durationMs: number,
|
|
176
|
+
): void {
|
|
177
|
+
p.tokens = Math.max(p.tokens, tokens);
|
|
178
|
+
p.toolUses = Math.max(p.toolUses, toolUses);
|
|
179
|
+
if (durationMs > p.durationMs) p.durationMs = durationMs;
|
|
180
|
+
}
|
|
181
|
+
|
|
155
182
|
/** Mirror a progress update from runAgent into a TaskProgress row.
|
|
156
183
|
*
|
|
157
184
|
* Runner callbacks report attempt-local counters. Offset/merge them so a single
|
|
@@ -162,12 +189,12 @@ export function updateProgressFromRun(
|
|
|
162
189
|
u: AgentProgressUpdate,
|
|
163
190
|
offsets: RunUpdateOffset = {},
|
|
164
191
|
): void {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
192
|
+
bumpProgressCounters(
|
|
193
|
+
p,
|
|
194
|
+
(offsets.tokensOffset ?? 0) + u.tokens,
|
|
195
|
+
(offsets.toolUsesOffset ?? 0) + u.toolUses,
|
|
196
|
+
u.durationMs,
|
|
197
|
+
);
|
|
171
198
|
p.lastActivityAt = u.lastActivityAt;
|
|
172
199
|
p.activities = mergeToolActivities(p.activities, u.activities);
|
|
173
200
|
p.failureKind = u.failureKind;
|
|
@@ -181,38 +208,66 @@ function updateProgressFromResult(p: TaskProgress, r: TaskResult): void {
|
|
|
181
208
|
p.failureKind = r.failureKind;
|
|
182
209
|
}
|
|
183
210
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
*
|
|
187
|
-
|
|
211
|
+
/** Outcome of one logical task run: the final result plus how many same-model
|
|
212
|
+
* retries it took. Telemetry is recorded by the caller at the outermost
|
|
213
|
+
* lifecycle boundary (see recordTaskOutcome). */
|
|
214
|
+
interface TaskOutcome {
|
|
215
|
+
result: TaskResult;
|
|
216
|
+
retries: number;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Notify the optional status observer without making UI/progress delivery part
|
|
220
|
+
* of task correctness. In particular, a host callback must not replace a task
|
|
221
|
+
* result or skip the outer telemetry write. */
|
|
222
|
+
function notifyStatusChange(env: TaskRunEnv): void {
|
|
223
|
+
if (!env.onStatusChange) return;
|
|
224
|
+
try {
|
|
225
|
+
env.onStatusChange();
|
|
226
|
+
} catch (error) {
|
|
227
|
+
console.error("[delegate] task status callback threw; continuing", error);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
188
230
|
|
|
189
231
|
/** Apply a TaskResult to progress and notify the env (sync fires onUpdate).
|
|
190
232
|
* Used at every return point in runResolvedTask — mirrors the old fire() pattern
|
|
191
|
-
* that the duplicated sync/async bodies used after every early-return.
|
|
233
|
+
* that the duplicated sync/async bodies used after every early-return.
|
|
234
|
+
* Telemetry is NOT recorded here: the scratch wrapper may still rewrite the
|
|
235
|
+
* result (path mapping, cleanup errors), so recording happens exactly once,
|
|
236
|
+
* in runResolvedTaskUnlocked, on the final object. */
|
|
192
237
|
function finishTask(
|
|
193
238
|
env: TaskRunEnv,
|
|
194
239
|
p: TaskProgress,
|
|
195
240
|
r: TaskResult,
|
|
196
|
-
task: ResolvedTask,
|
|
197
241
|
retries = 0,
|
|
198
|
-
):
|
|
242
|
+
): TaskOutcome {
|
|
199
243
|
updateProgressFromResult(p, r);
|
|
200
|
-
|
|
244
|
+
notifyStatusChange(env);
|
|
245
|
+
return { result: r, retries };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Record the telemetry task row at the outermost lifecycle boundary — once
|
|
249
|
+
* per runResolvedTask, on the final (post-scratch-wrap) result. Returns the
|
|
250
|
+
* result so call sites stay flat. */
|
|
251
|
+
function recordTaskOutcome(
|
|
252
|
+
env: TaskRunEnv,
|
|
253
|
+
p: TaskProgress,
|
|
254
|
+
task: ResolvedTask,
|
|
255
|
+
outcome: TaskOutcome,
|
|
256
|
+
): TaskResult {
|
|
201
257
|
if (env.telemetryCallId) {
|
|
202
|
-
|
|
258
|
+
recordTask({
|
|
203
259
|
callId: env.telemetryCallId,
|
|
204
260
|
generation: env.telemetryGeneration,
|
|
261
|
+
telemetryConfig: env.telemetryConfig,
|
|
205
262
|
async: env.async ?? false,
|
|
206
263
|
taskIndex: p.index,
|
|
207
264
|
task,
|
|
208
265
|
progress: p,
|
|
209
|
-
result:
|
|
210
|
-
retries,
|
|
266
|
+
result: outcome.result,
|
|
267
|
+
retries: outcome.retries,
|
|
211
268
|
});
|
|
212
|
-
if (id) taskTelemetryIds.set(r, id);
|
|
213
269
|
}
|
|
214
|
-
|
|
215
|
-
return r;
|
|
270
|
+
return outcome.result;
|
|
216
271
|
}
|
|
217
272
|
|
|
218
273
|
/** A failure attributable to the resolved model/provider — not transient for
|
|
@@ -335,7 +390,7 @@ async function sleepForWholeTaskRetry(
|
|
|
335
390
|
async function buildDelegateSession(
|
|
336
391
|
task: ResolvedTask,
|
|
337
392
|
sessionManager: SessionManager,
|
|
338
|
-
|
|
393
|
+
env: TaskRunEnv,
|
|
339
394
|
): Promise<AgentSession> {
|
|
340
395
|
// Resolve host deps for this task's cwd + system prompt. Extension-free
|
|
341
396
|
// resource loaders are cached after the first call; provider-configured or
|
|
@@ -346,7 +401,7 @@ async function buildDelegateSession(
|
|
|
346
401
|
// Pass only the provider needed by this task. This keeps a non-Kilo task
|
|
347
402
|
// from receiving Kilo's provider/auth adapter merely because Kilo is also
|
|
348
403
|
// configured in the parent runtime.
|
|
349
|
-
const providerConfig = modelRegistry.getRegisteredProviderConfig?.(
|
|
404
|
+
const providerConfig = env.modelRegistry.getRegisteredProviderConfig?.(
|
|
350
405
|
task.model.provider,
|
|
351
406
|
);
|
|
352
407
|
const providerConfigs = providerConfig
|
|
@@ -357,6 +412,10 @@ async function buildDelegateSession(
|
|
|
357
412
|
systemPrompt: task.systemPrompt,
|
|
358
413
|
providerConfigs,
|
|
359
414
|
modelProvider: task.model.provider,
|
|
415
|
+
// Freeze the provider-extension allowlist to the dispatch-scoped snapshot
|
|
416
|
+
// so a later delegate.json edit cannot change which executable code an
|
|
417
|
+
// already-spawned async worker is allowed to load.
|
|
418
|
+
delegateConfig: env.config,
|
|
360
419
|
});
|
|
361
420
|
|
|
362
421
|
const { session } = await createAgentSession({
|
|
@@ -378,126 +437,128 @@ async function buildDelegateSession(
|
|
|
378
437
|
return session;
|
|
379
438
|
}
|
|
380
439
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
440
|
+
type AcquireResult = AcquiredSession | { error: TaskResult };
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Try a live pooled session. Returns a hit, a formatted mismatch/conflict
|
|
444
|
+
* error, or `undefined` on miss so the caller can resume or create fresh.
|
|
445
|
+
* checkout is pure (no lastUsed bump); lastUsed is bumped by commit().
|
|
446
|
+
*/
|
|
447
|
+
function checkoutPooledSession(
|
|
384
448
|
task: ResolvedTask,
|
|
385
449
|
p: TaskProgress,
|
|
386
|
-
):
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
if (
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
`Session '${task.sessionId}' config mismatch. Close and recreate: ${detail}`,
|
|
410
|
-
),
|
|
411
|
-
};
|
|
412
|
-
}
|
|
413
|
-
if (co.status === "hit") {
|
|
414
|
-
// A pooled session has its own accumulated context — resumeFrom pointing
|
|
415
|
-
// elsewhere is contradictory. This folds the old defensive agentPool.has
|
|
416
|
-
// precheck: checkout already told us the session is live.
|
|
417
|
-
if (task.resumeFrom) {
|
|
418
|
-
return {
|
|
419
|
-
error: failTask(
|
|
420
|
-
task,
|
|
421
|
-
`resumeFrom conflicts with active sessionId '${task.sessionId}'. The pooled session has its own accumulated context. Close the session first if you want to resume from a different point.`,
|
|
422
|
-
),
|
|
423
|
-
};
|
|
424
|
-
}
|
|
425
|
-
p.model = co.modelId;
|
|
426
|
-
return {
|
|
427
|
-
session: co.session,
|
|
428
|
-
sessionManager: co.sessionManager,
|
|
429
|
-
sessionFile: co.sessionFile,
|
|
430
|
-
lifecycleOwnsSession: false,
|
|
431
|
-
};
|
|
432
|
-
}
|
|
433
|
-
// status === "miss" → fall through to resume / fresh materialization.
|
|
450
|
+
): AcquireResult | undefined {
|
|
451
|
+
if (!task.sessionId) return undefined;
|
|
452
|
+
const co = pool.checkout(task.sessionId, {
|
|
453
|
+
cwd: task.cwd,
|
|
454
|
+
thinking: task.thinking,
|
|
455
|
+
tools: task.tools,
|
|
456
|
+
providerExtensions: task.providerExtensionSources ?? "",
|
|
457
|
+
...task.reuseIntent,
|
|
458
|
+
});
|
|
459
|
+
if (co.status === "mismatch") {
|
|
460
|
+
const detail = co.mismatches
|
|
461
|
+
.map((m) =>
|
|
462
|
+
m.field === "providerExtensions"
|
|
463
|
+
? "providerExtensions: changed"
|
|
464
|
+
: `${m.field}: '${m.frozen}' vs '${m.requested}'`,
|
|
465
|
+
)
|
|
466
|
+
.join("; ");
|
|
467
|
+
return {
|
|
468
|
+
error: failTask(
|
|
469
|
+
task,
|
|
470
|
+
`Session '${task.sessionId}' config mismatch. Close and recreate: ${detail}`,
|
|
471
|
+
),
|
|
472
|
+
};
|
|
434
473
|
}
|
|
435
|
-
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
// concrete prior conversation we must continue, whereas a sessionId miss just
|
|
439
|
-
// means "create a new pooled session under this id".
|
|
474
|
+
if (co.status !== "hit") return undefined;
|
|
475
|
+
// A pooled session has its own accumulated context — resumeFrom pointing
|
|
476
|
+
// elsewhere is contradictory.
|
|
440
477
|
if (task.resumeFrom) {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
resolvedPath,
|
|
457
|
-
),
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
// Open the existing session and let createAgentSession restore its messages
|
|
461
|
-
// internally (sdk.js reads buildSessionContext().messages + model/thinking).
|
|
462
|
-
let resumed: SessionManager;
|
|
463
|
-
try {
|
|
464
|
-
resumed = SessionManager.open(resolvedPath);
|
|
465
|
-
} catch {
|
|
466
|
-
return {
|
|
467
|
-
error: failTask(
|
|
468
|
-
task,
|
|
469
|
-
`resumeFrom: corrupt session: ${resolvedPath}`,
|
|
470
|
-
resolvedPath,
|
|
471
|
-
),
|
|
472
|
-
};
|
|
473
|
-
}
|
|
474
|
-
// Non-empty sessions have at least the header + the restored branch. An
|
|
475
|
-
// empty/corrupt file surfaces as a session with no restorable messages.
|
|
476
|
-
if (!resumed.buildSessionContext().messages.length) {
|
|
477
|
-
return {
|
|
478
|
-
error: failTask(
|
|
479
|
-
task,
|
|
480
|
-
`resumeFrom: empty session: ${resolvedPath}`,
|
|
481
|
-
resolvedPath,
|
|
482
|
-
),
|
|
483
|
-
};
|
|
484
|
-
}
|
|
478
|
+
return {
|
|
479
|
+
error: failTask(
|
|
480
|
+
task,
|
|
481
|
+
`resumeFrom conflicts with active sessionId '${task.sessionId}'. The pooled session has its own accumulated context. Close the session first if you want to resume from a different point.`,
|
|
482
|
+
),
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
p.model = co.modelId;
|
|
486
|
+
return {
|
|
487
|
+
session: co.session,
|
|
488
|
+
sessionManager: co.sessionManager,
|
|
489
|
+
sessionFile: co.sessionFile,
|
|
490
|
+
lifecycleOwnsSession: false,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
485
493
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
494
|
+
/** Resume takes precedence over a fresh sessionId miss: resumeFrom points at a
|
|
495
|
+
* concrete prior conversation we must continue. */
|
|
496
|
+
async function resumeFromSessionFile(
|
|
497
|
+
env: TaskRunEnv,
|
|
498
|
+
task: ResolvedTask,
|
|
499
|
+
resumeFrom: string,
|
|
500
|
+
): Promise<AcquireResult> {
|
|
501
|
+
const resumeFromPathError = validateResumeFromPath(resumeFrom);
|
|
502
|
+
if (resumeFromPathError) {
|
|
503
|
+
return {
|
|
504
|
+
error: failTask(
|
|
505
|
+
task,
|
|
506
|
+
`resumeFrom: invalid session path: ${resumeFromPathError}; got ${JSON.stringify(resumeFrom)}`,
|
|
507
|
+
),
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
const resolvedPath = resolveCwd(resumeFrom);
|
|
511
|
+
if (!fs.existsSync(resolvedPath)) {
|
|
512
|
+
return {
|
|
513
|
+
error: failTask(
|
|
514
|
+
task,
|
|
515
|
+
`resumeFrom: file not found: ${resolvedPath}`,
|
|
516
|
+
resolvedPath,
|
|
517
|
+
),
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
// Open the existing session and let createAgentSession restore its messages
|
|
521
|
+
// internally (sdk.js reads buildSessionContext().messages + model/thinking).
|
|
522
|
+
let resumed: SessionManager;
|
|
523
|
+
try {
|
|
524
|
+
resumed = SessionManager.open(resolvedPath);
|
|
525
|
+
} catch {
|
|
491
526
|
return {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
527
|
+
error: failTask(
|
|
528
|
+
task,
|
|
529
|
+
`resumeFrom: corrupt session: ${resolvedPath}`,
|
|
530
|
+
resolvedPath,
|
|
531
|
+
),
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
// Non-empty sessions have at least the header + the restored branch. An
|
|
535
|
+
// empty/corrupt file surfaces as a session with no restorable messages.
|
|
536
|
+
if (!resumed.buildSessionContext().messages.length) {
|
|
537
|
+
return {
|
|
538
|
+
error: failTask(
|
|
539
|
+
task,
|
|
540
|
+
`resumeFrom: empty session: ${resolvedPath}`,
|
|
541
|
+
resolvedPath,
|
|
542
|
+
),
|
|
496
543
|
};
|
|
497
544
|
}
|
|
498
545
|
|
|
499
|
-
|
|
500
|
-
|
|
546
|
+
const session = await buildDelegateSession(task, resumed, env);
|
|
547
|
+
return {
|
|
548
|
+
session,
|
|
549
|
+
sessionManager: resumed,
|
|
550
|
+
sessionFile: resolvedPath,
|
|
551
|
+
lifecycleOwnsSession: true,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
async function createFreshSession(
|
|
556
|
+
env: TaskRunEnv,
|
|
557
|
+
task: ResolvedTask,
|
|
558
|
+
): Promise<AcquireResult> {
|
|
559
|
+
let sessionManager: SessionManager;
|
|
560
|
+
let sessionFile: string | undefined;
|
|
561
|
+
if (task.workspace === "scratch" || task.workspace === "isolated") {
|
|
501
562
|
// A discarded filesystem must not advertise a resumable conversation: a
|
|
502
563
|
// later resume would run against the source cwd and silently lose scratch
|
|
503
564
|
// isolation. Keep scratch transcripts in memory only.
|
|
@@ -513,11 +574,7 @@ async function acquireAgentSession(
|
|
|
513
574
|
sessionFile = fresh.file;
|
|
514
575
|
}
|
|
515
576
|
|
|
516
|
-
const session = await buildDelegateSession(
|
|
517
|
-
task,
|
|
518
|
-
sessionManager,
|
|
519
|
-
env.modelRegistry,
|
|
520
|
-
);
|
|
577
|
+
const session = await buildDelegateSession(task, sessionManager, env);
|
|
521
578
|
return {
|
|
522
579
|
session,
|
|
523
580
|
sessionManager,
|
|
@@ -526,6 +583,20 @@ async function acquireAgentSession(
|
|
|
526
583
|
};
|
|
527
584
|
}
|
|
528
585
|
|
|
586
|
+
/** Resolve the agent + session for a task. Pool hit / resume / fresh. */
|
|
587
|
+
async function acquireAgentSession(
|
|
588
|
+
env: TaskRunEnv,
|
|
589
|
+
task: ResolvedTask,
|
|
590
|
+
p: TaskProgress,
|
|
591
|
+
): Promise<AcquireResult> {
|
|
592
|
+
if (task.sessionId) {
|
|
593
|
+
const pooled = checkoutPooledSession(task, p);
|
|
594
|
+
if (pooled) return pooled;
|
|
595
|
+
}
|
|
596
|
+
if (task.resumeFrom) return resumeFromSessionFile(env, task, task.resumeFrom);
|
|
597
|
+
return createFreshSession(env, task);
|
|
598
|
+
}
|
|
599
|
+
|
|
529
600
|
/**
|
|
530
601
|
* Resolve the `sessionFile` to report on a TaskResult.
|
|
531
602
|
*
|
|
@@ -574,18 +645,41 @@ async function runResolvedTaskUnlocked(
|
|
|
574
645
|
p: TaskProgress,
|
|
575
646
|
taskIndex: number,
|
|
576
647
|
): Promise<TaskResult> {
|
|
648
|
+
if (
|
|
649
|
+
task.workspace === "isolated" &&
|
|
650
|
+
(task.sessionId || task.resumeFrom || task.sessionAction)
|
|
651
|
+
) {
|
|
652
|
+
return recordTaskOutcome(
|
|
653
|
+
env,
|
|
654
|
+
p,
|
|
655
|
+
task,
|
|
656
|
+
finishTask(
|
|
657
|
+
env,
|
|
658
|
+
p,
|
|
659
|
+
failTask(
|
|
660
|
+
task,
|
|
661
|
+
"workspace 'isolated' is one-shot and cannot be combined with sessionId, resumeFrom, or sessionAction.",
|
|
662
|
+
),
|
|
663
|
+
),
|
|
664
|
+
);
|
|
665
|
+
}
|
|
577
666
|
if (task.workspace !== "scratch") {
|
|
578
|
-
|
|
667
|
+
const outcome = await runResolvedTaskCore(env, task, p, taskIndex);
|
|
668
|
+
return recordTaskOutcome(env, p, task, outcome);
|
|
579
669
|
}
|
|
580
670
|
if (task.sessionId || task.resumeFrom || task.sessionAction) {
|
|
581
|
-
return
|
|
671
|
+
return recordTaskOutcome(
|
|
582
672
|
env,
|
|
583
673
|
p,
|
|
584
|
-
failTask(
|
|
585
|
-
task,
|
|
586
|
-
"workspace 'scratch' is one-shot and cannot be combined with sessionId, resumeFrom, or sessionAction.",
|
|
587
|
-
),
|
|
588
674
|
task,
|
|
675
|
+
finishTask(
|
|
676
|
+
env,
|
|
677
|
+
p,
|
|
678
|
+
failTask(
|
|
679
|
+
task,
|
|
680
|
+
"workspace 'scratch' is one-shot and cannot be combined with sessionId, resumeFrom, or sessionAction.",
|
|
681
|
+
),
|
|
682
|
+
),
|
|
589
683
|
);
|
|
590
684
|
}
|
|
591
685
|
|
|
@@ -596,15 +690,20 @@ async function runResolvedTaskUnlocked(
|
|
|
596
690
|
: undefined;
|
|
597
691
|
let workspace: Awaited<ReturnType<typeof createScratchWorkspace>>;
|
|
598
692
|
try {
|
|
599
|
-
workspace = await
|
|
693
|
+
workspace = await createScratchWorkspaceForTesting(
|
|
694
|
+
task.cwd,
|
|
695
|
+
env.signal,
|
|
696
|
+
deadlineAt,
|
|
697
|
+
);
|
|
600
698
|
} catch (error) {
|
|
601
699
|
const setupError = error instanceof Error ? error.message : String(error);
|
|
602
700
|
const deadlineExceeded =
|
|
603
701
|
!env.signal?.aborted && error instanceof ScratchDeadlineError;
|
|
604
|
-
return
|
|
702
|
+
return recordTaskOutcome(
|
|
605
703
|
env,
|
|
606
704
|
p,
|
|
607
|
-
|
|
705
|
+
task,
|
|
706
|
+
finishTask(env, p, {
|
|
608
707
|
...failTask(
|
|
609
708
|
task,
|
|
610
709
|
env.signal?.aborted
|
|
@@ -615,8 +714,7 @@ async function runResolvedTaskUnlocked(
|
|
|
615
714
|
),
|
|
616
715
|
failureKind: deadlineExceeded ? "deadline_exceeded" : undefined,
|
|
617
716
|
durationMs: Date.now() - startedAt,
|
|
618
|
-
},
|
|
619
|
-
task,
|
|
717
|
+
}),
|
|
620
718
|
);
|
|
621
719
|
}
|
|
622
720
|
|
|
@@ -624,27 +722,18 @@ async function runResolvedTaskUnlocked(
|
|
|
624
722
|
...task,
|
|
625
723
|
cwd: workspace.cwd,
|
|
626
724
|
};
|
|
725
|
+
let outcome: TaskOutcome | undefined;
|
|
627
726
|
let result: TaskResult | undefined;
|
|
628
727
|
let cleanupError: string | undefined;
|
|
629
728
|
let needsCorrection = false;
|
|
630
|
-
let telemetryTaskId: string | undefined;
|
|
631
|
-
let retries = 0;
|
|
632
729
|
try {
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
deadlineAt,
|
|
641
|
-
},
|
|
642
|
-
);
|
|
643
|
-
result = preMappingResult;
|
|
644
|
-
// Capture metadata before scratch wrapping creates a new result object.
|
|
645
|
-
// Both values belong to the logical task and must survive that clone.
|
|
646
|
-
telemetryTaskId = taskTelemetryIds.get(result);
|
|
647
|
-
retries = taskRetryCounts.get(result) ?? 0;
|
|
730
|
+
outcome = await runResolvedTaskCore(env, executionTask, p, taskIndex, {
|
|
731
|
+
taskStartedAt: startedAt,
|
|
732
|
+
deadlineAt,
|
|
733
|
+
});
|
|
734
|
+
// Keep the pre-mapping result in `result` so the catch below can preserve
|
|
735
|
+
// its paid-for counters if path mapping throws mid-rewrite.
|
|
736
|
+
result = outcome.result;
|
|
648
737
|
result = {
|
|
649
738
|
...result,
|
|
650
739
|
workspace: "scratch",
|
|
@@ -676,8 +765,8 @@ async function runResolvedTaskUnlocked(
|
|
|
676
765
|
workspace: "scratch",
|
|
677
766
|
durationMs: Date.now() - startedAt,
|
|
678
767
|
};
|
|
679
|
-
// runResolvedTaskCore may already have
|
|
680
|
-
//
|
|
768
|
+
// runResolvedTaskCore may already have notified a successful result
|
|
769
|
+
// before path mapping failed. Correct that observable outcome below.
|
|
681
770
|
needsCorrection = true;
|
|
682
771
|
} finally {
|
|
683
772
|
try {
|
|
@@ -706,390 +795,481 @@ async function runResolvedTaskUnlocked(
|
|
|
706
795
|
durationMs: Math.max(result.durationMs, Date.now() - startedAt),
|
|
707
796
|
};
|
|
708
797
|
updateProgressFromResult(p, result);
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
798
|
+
notifyStatusChange(env);
|
|
799
|
+
}
|
|
800
|
+
// Single telemetry write for the whole logical task — after scratch
|
|
801
|
+
// wrapping and cleanup, on the result actually returned, so no correction
|
|
802
|
+
// upsert is ever needed.
|
|
803
|
+
return recordTaskOutcome(env, p, task, {
|
|
804
|
+
result,
|
|
805
|
+
retries: outcome?.retries ?? 0,
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
interface AttemptTiming {
|
|
810
|
+
taskStartedAt: number;
|
|
811
|
+
deadlineAt: number | undefined;
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
interface AttemptAccounting {
|
|
815
|
+
hasBashExecution: boolean;
|
|
816
|
+
cumulativeTokens: number;
|
|
817
|
+
cumulativeToolUses: number;
|
|
818
|
+
accumulatedUsage: ReturnType<typeof emptyUsage>;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function resolveAttemptTiming(
|
|
822
|
+
task: ResolvedTask,
|
|
823
|
+
timing?: AttemptTiming,
|
|
824
|
+
): AttemptTiming {
|
|
825
|
+
const taskStartedAt = timing?.taskStartedAt ?? Date.now();
|
|
826
|
+
return {
|
|
827
|
+
taskStartedAt,
|
|
828
|
+
deadlineAt:
|
|
829
|
+
timing?.deadlineAt ??
|
|
830
|
+
(task.deadlineMs && task.deadlineMs > 0
|
|
831
|
+
? taskStartedAt + task.deadlineMs
|
|
832
|
+
: undefined),
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/** Close/list short-circuits. Returns undefined when the task should prompt. */
|
|
837
|
+
async function applySessionAction(
|
|
838
|
+
env: TaskRunEnv,
|
|
839
|
+
task: ResolvedTask,
|
|
840
|
+
p: TaskProgress,
|
|
841
|
+
): Promise<TaskOutcome | undefined> {
|
|
842
|
+
if (task.sessionAction === "close") {
|
|
843
|
+
if (!task.sessionId) {
|
|
844
|
+
return finishTask(
|
|
845
|
+
env,
|
|
846
|
+
p,
|
|
847
|
+
failTask(task, "sessionAction='close' requires sessionId."),
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
// The per-session lock for action-based operations is already held by the
|
|
851
|
+
// outer runResolvedTask() wrapper. Use the internal close helper to avoid a
|
|
852
|
+
// reentrant deadlock on the same key.
|
|
853
|
+
const closed = await pool._closePooledAgentWithoutLock(task.sessionId);
|
|
854
|
+
return finishTask(
|
|
855
|
+
env,
|
|
856
|
+
p,
|
|
857
|
+
completeSessionAction(
|
|
858
|
+
task,
|
|
859
|
+
closed
|
|
860
|
+
? `Session '${task.sessionId}' closed.`
|
|
861
|
+
: `Session '${task.sessionId}' not found.`,
|
|
862
|
+
Date.now() - env.delegateStartedAt,
|
|
863
|
+
),
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
if (task.sessionAction === "list") {
|
|
868
|
+
return finishTask(
|
|
869
|
+
env,
|
|
870
|
+
p,
|
|
871
|
+
completeSessionAction(
|
|
719
872
|
task,
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
873
|
+
`Active sessions:\n${pool.listPooledAgents().join("\n")}`,
|
|
874
|
+
Date.now() - env.delegateStartedAt,
|
|
875
|
+
),
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
return undefined;
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
function busySessionConflict(
|
|
882
|
+
env: TaskRunEnv,
|
|
883
|
+
task: ResolvedTask,
|
|
884
|
+
): TaskResult | undefined {
|
|
885
|
+
if (!task.sessionId) return undefined;
|
|
886
|
+
const busyTicketId = isSessionBusy(task.sessionId);
|
|
887
|
+
if (busyTicketId && busyTicketId !== env.ticketId) {
|
|
888
|
+
return failTask(
|
|
889
|
+
task,
|
|
890
|
+
`Session '${task.sessionId}' is already in use by ticket ${busyTicketId}. Each session can only handle one task at a time.`,
|
|
891
|
+
);
|
|
892
|
+
}
|
|
893
|
+
return undefined;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function deadlineExceededResult(
|
|
897
|
+
task: ResolvedTask,
|
|
898
|
+
timing: AttemptTiming,
|
|
899
|
+
prior?: TaskResult,
|
|
900
|
+
): TaskResult {
|
|
901
|
+
const budgetMs = Math.max(0, (timing.deadlineAt ?? 0) - timing.taskStartedAt);
|
|
902
|
+
return {
|
|
903
|
+
id: task.id,
|
|
904
|
+
agent: task.agentName,
|
|
905
|
+
output: prior?.output ?? "",
|
|
906
|
+
error: formatDeadlineExceededError(budgetMs),
|
|
907
|
+
failureKind: "deadline_exceeded",
|
|
908
|
+
durationMs: prior?.durationMs ?? 0,
|
|
909
|
+
tokens: prior?.tokens ?? 0,
|
|
910
|
+
usage: prior?.usage ?? emptyUsage(),
|
|
911
|
+
sessionFile: prior?.sessionFile,
|
|
912
|
+
touchedFiles: prior?.touchedFiles ?? [],
|
|
913
|
+
attributedFiles: prior?.attributedFiles ?? [],
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
function noteAttemptProgress(
|
|
918
|
+
env: TaskRunEnv,
|
|
919
|
+
p: TaskProgress,
|
|
920
|
+
u: AgentProgressUpdate,
|
|
921
|
+
timing: AttemptTiming,
|
|
922
|
+
accounting: AttemptAccounting,
|
|
923
|
+
): void {
|
|
924
|
+
if (
|
|
925
|
+
!accounting.hasBashExecution &&
|
|
926
|
+
u.activities.some((activity) => activity.name === "bash")
|
|
927
|
+
) {
|
|
928
|
+
accounting.hasBashExecution = true;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
const mapped: AgentProgressUpdate = {
|
|
932
|
+
...u,
|
|
933
|
+
tokens: accounting.cumulativeTokens + u.tokens,
|
|
934
|
+
toolUses: accounting.cumulativeToolUses + u.toolUses,
|
|
935
|
+
durationMs: Date.now() - timing.taskStartedAt,
|
|
936
|
+
};
|
|
937
|
+
|
|
938
|
+
// Keep live totals monotonic across attempts.
|
|
939
|
+
bumpProgressCounters(p, mapped.tokens, mapped.toolUses, mapped.durationMs);
|
|
940
|
+
env.onProgress(p, mapped);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/** Commit, record, or evict a pooled session after one prompt attempt. */
|
|
944
|
+
async function settlePooledAttempt(
|
|
945
|
+
task: ResolvedTask,
|
|
946
|
+
acquired: AcquiredSession,
|
|
947
|
+
r: {
|
|
948
|
+
error?: string;
|
|
949
|
+
failureKind?: TaskResult["failureKind"];
|
|
950
|
+
prompted?: boolean;
|
|
951
|
+
tokens: number;
|
|
952
|
+
},
|
|
953
|
+
sessionReleased: boolean,
|
|
954
|
+
): Promise<boolean> {
|
|
955
|
+
if (!task.sessionId) return sessionReleased;
|
|
956
|
+
if (acquired.lifecycleOwnsSession) {
|
|
957
|
+
// Pool misses (including resumeFrom) transfer ownership only on
|
|
958
|
+
// successful completion; failures are owned by lifecycle and must
|
|
959
|
+
// be disposed in this finally path.
|
|
960
|
+
if (
|
|
961
|
+
!r.error &&
|
|
962
|
+
r.failureKind !== "stalled" &&
|
|
963
|
+
r.failureKind !== "deadline_exceeded"
|
|
964
|
+
) {
|
|
965
|
+
const committed = pool.commit(task.sessionId, {
|
|
966
|
+
session: acquired.session,
|
|
967
|
+
sessionManager: acquired.sessionManager,
|
|
968
|
+
sessionFile: acquired.sessionFile,
|
|
969
|
+
frozen: {
|
|
970
|
+
systemPrompt: task.systemPrompt,
|
|
971
|
+
model: task.model,
|
|
972
|
+
thinking: task.thinking,
|
|
973
|
+
tools: task.tools,
|
|
974
|
+
cwd: task.cwd,
|
|
975
|
+
providerExtensions: task.providerExtensionSources ?? "",
|
|
976
|
+
},
|
|
977
|
+
tokens: r.tokens,
|
|
723
978
|
});
|
|
979
|
+
return sessionReleased || committed;
|
|
724
980
|
}
|
|
725
|
-
|
|
981
|
+
return sessionReleased;
|
|
726
982
|
}
|
|
727
|
-
|
|
983
|
+
|
|
984
|
+
// A stalled, parent-aborted, or mid-prompt deadline-exceeded pooled
|
|
985
|
+
// attempt is not safe to keep; the session may have been mutated.
|
|
986
|
+
// A pre-prompt deadline (runner never called session.prompt()) left
|
|
987
|
+
// the session in its pre-task state, so return it to the pool intact.
|
|
988
|
+
if (
|
|
989
|
+
r.failureKind === "stalled" ||
|
|
990
|
+
r.error === "Aborted" ||
|
|
991
|
+
(r.failureKind === "deadline_exceeded" && r.prompted !== false)
|
|
992
|
+
) {
|
|
993
|
+
try {
|
|
994
|
+
return (
|
|
995
|
+
(await pool._closePooledAgentWithoutLock(task.sessionId)) ||
|
|
996
|
+
sessionReleased
|
|
997
|
+
);
|
|
998
|
+
} catch (error) {
|
|
999
|
+
// Preserve the primary failure result while logging the cleanup
|
|
1000
|
+
// failure explicitly. A pooled session may still be removed by
|
|
1001
|
+
// the pool; a pool-miss remains lifecycle-owned and is handled
|
|
1002
|
+
// by the finally path above.
|
|
1003
|
+
console.error(
|
|
1004
|
+
`[delegate] failed to dispose aborted, stalled, or deadline-exceeded pooled session '${task.sessionId}'`,
|
|
1005
|
+
error,
|
|
1006
|
+
);
|
|
1007
|
+
return sessionReleased;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
if (r.failureKind !== "deadline_exceeded") {
|
|
1011
|
+
// Pool hits stay owned by the pool, and non-stalled, non-aborted
|
|
1012
|
+
// completions (including failed attempts) must still count usage.
|
|
1013
|
+
pool.recordUse(task.sessionId, r.tokens);
|
|
1014
|
+
}
|
|
1015
|
+
// Pre-prompt deadline (prompted === false): the pooled session was
|
|
1016
|
+
// checked out but never used. Leave it in the pool with no usage
|
|
1017
|
+
// recorded.
|
|
1018
|
+
return sessionReleased;
|
|
728
1019
|
}
|
|
729
1020
|
|
|
730
|
-
|
|
1021
|
+
/** Acquire, prompt, and settle one attempt. Mutates `accounting` on success. */
|
|
1022
|
+
async function runTaskAttempt(
|
|
731
1023
|
env: TaskRunEnv,
|
|
732
1024
|
task: ResolvedTask,
|
|
733
1025
|
p: TaskProgress,
|
|
734
|
-
|
|
735
|
-
|
|
1026
|
+
timing: AttemptTiming,
|
|
1027
|
+
accounting: AttemptAccounting,
|
|
736
1028
|
): Promise<TaskResult> {
|
|
1029
|
+
let attemptToolUsesObserved = 0;
|
|
1030
|
+
const onProgress = (u: AgentProgressUpdate): void => {
|
|
1031
|
+
attemptToolUsesObserved = Math.max(attemptToolUsesObserved, u.toolUses);
|
|
1032
|
+
noteAttemptProgress(env, p, u, timing, accounting);
|
|
1033
|
+
};
|
|
1034
|
+
|
|
1035
|
+
const acquired = await acquireAgentSession(env, task, p);
|
|
1036
|
+
if ("error" in acquired) return acquired.error;
|
|
1037
|
+
|
|
1038
|
+
// A pool hit is already owned by the pool. Fresh/resumed sessions belong
|
|
1039
|
+
// to this attempt until commit/recordUse/close logic runs.
|
|
1040
|
+
let sessionReleased = !acquired.lifecycleOwnsSession;
|
|
737
1041
|
try {
|
|
738
|
-
//
|
|
1042
|
+
// Re-check abort after acquisition. The pre-acquire check at the top can
|
|
1043
|
+
// miss a signal that fires during getHostDeps/createAgentSession/git
|
|
1044
|
+
// baseline. runAgentSession re-checks after attaching its listener, but a
|
|
1045
|
+
// cancelled ticket should not even start the subagent (no file writes, no
|
|
1046
|
+
// pool insert).
|
|
739
1047
|
if (env.signal?.aborted) {
|
|
740
|
-
return
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
// ── Session busy guard (defense-in-depth) ────────────────────────
|
|
744
|
-
// Primary validation is in execute() before ticket creation.
|
|
745
|
-
// This catches edge cases where validation missed a conflict.
|
|
746
|
-
if (task.sessionId) {
|
|
747
|
-
const busyTicketId = isSessionBusy(task.sessionId);
|
|
748
|
-
if (busyTicketId && busyTicketId !== env.ticketId) {
|
|
749
|
-
const msg = `Session '${task.sessionId}' is already in use by ticket ${busyTicketId}. Each session can only handle one task at a time.`;
|
|
750
|
-
return finishTask(env, p, failTask(task, msg), task);
|
|
751
|
-
}
|
|
1048
|
+
return failTask(task, "Aborted");
|
|
752
1049
|
}
|
|
753
1050
|
|
|
754
|
-
|
|
755
|
-
|
|
1051
|
+
// Snapshot git status before the run so touchedFiles can diff after.
|
|
1052
|
+
// AgentSession owns retry/compaction internally — runAgentSession just
|
|
1053
|
+
// drives the prompt and maps events to the progress model. Git failures
|
|
1054
|
+
// degrade to an undefined baseline, which tells the runner to skip
|
|
1055
|
+
// git-based attribution entirely; see getGitChangedFiles for the
|
|
1056
|
+
// contract.
|
|
1057
|
+
const gitBaseline = await getGitChangedFiles(task.cwd);
|
|
1058
|
+
let r = await runAgentSessionForTesting(
|
|
1059
|
+
acquired.session,
|
|
1060
|
+
task.prompt,
|
|
1061
|
+
{ cwd: task.cwd },
|
|
1062
|
+
env.signal,
|
|
1063
|
+
onProgress,
|
|
1064
|
+
gitBaseline,
|
|
1065
|
+
timing.taskStartedAt,
|
|
1066
|
+
timing.deadlineAt,
|
|
1067
|
+
env.config,
|
|
1068
|
+
);
|
|
756
1069
|
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
failTask(task, "sessionAction='close' requires sessionId."),
|
|
764
|
-
task,
|
|
765
|
-
);
|
|
766
|
-
}
|
|
767
|
-
// The per-session lock for action-based operations is already held by the
|
|
768
|
-
// outer runResolvedTask() wrapper. Use the internal close helper to avoid a
|
|
769
|
-
// reentrant deadlock on the same key.
|
|
770
|
-
const closed = await pool._closePooledAgentWithoutLock(task.sessionId);
|
|
771
|
-
return finishTask(
|
|
772
|
-
env,
|
|
773
|
-
p,
|
|
774
|
-
completeSessionAction(
|
|
775
|
-
task,
|
|
776
|
-
closed
|
|
777
|
-
? `Session '${task.sessionId}' closed.`
|
|
778
|
-
: `Session '${task.sessionId}' not found.`,
|
|
779
|
-
Date.now() - env.delegateStartedAt,
|
|
780
|
-
),
|
|
781
|
-
task,
|
|
782
|
-
);
|
|
1070
|
+
accounting.cumulativeTokens += r.tokens;
|
|
1071
|
+
// The signal can fire after the pre-run check or while the runner is
|
|
1072
|
+
// collecting post-prompt evidence. Keep cancellation from looking like
|
|
1073
|
+
// success; finally below releases any uncommitted session.
|
|
1074
|
+
if (env.signal?.aborted && !r.error) {
|
|
1075
|
+
r = { ...r, error: "Aborted" };
|
|
783
1076
|
}
|
|
784
1077
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
task,
|
|
791
|
-
`Active sessions:\n${pool.listPooledAgents().join("\n")}`,
|
|
792
|
-
Date.now() - env.delegateStartedAt,
|
|
793
|
-
),
|
|
794
|
-
task,
|
|
795
|
-
);
|
|
796
|
-
}
|
|
1078
|
+
const sessionFile = resolveResumableSessionFile(
|
|
1079
|
+
acquired.sessionFile,
|
|
1080
|
+
acquired.sessionManager,
|
|
1081
|
+
r.error,
|
|
1082
|
+
);
|
|
797
1083
|
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
(task.deadlineMs && task.deadlineMs > 0
|
|
805
|
-
? taskStartedAt + task.deadlineMs
|
|
806
|
-
: undefined);
|
|
807
|
-
let accumulatedUsage = emptyUsage();
|
|
808
|
-
|
|
809
|
-
const onAttemptProgress = (u: AgentProgressUpdate): void => {
|
|
810
|
-
if (
|
|
811
|
-
!hasBashExecution &&
|
|
812
|
-
u.activities.some((activity) => activity.name === "bash")
|
|
813
|
-
) {
|
|
814
|
-
hasBashExecution = true;
|
|
815
|
-
}
|
|
1084
|
+
sessionReleased = await settlePooledAttempt(
|
|
1085
|
+
task,
|
|
1086
|
+
acquired,
|
|
1087
|
+
r,
|
|
1088
|
+
sessionReleased,
|
|
1089
|
+
);
|
|
816
1090
|
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
};
|
|
1091
|
+
accounting.accumulatedUsage = addUsage(
|
|
1092
|
+
accounting.accumulatedUsage,
|
|
1093
|
+
r.usage,
|
|
1094
|
+
);
|
|
1095
|
+
accounting.cumulativeToolUses += attemptToolUsesObserved;
|
|
823
1096
|
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
1097
|
+
return {
|
|
1098
|
+
id: task.id,
|
|
1099
|
+
agent: task.agentName,
|
|
1100
|
+
output: r.output,
|
|
1101
|
+
error: r.error,
|
|
1102
|
+
// Classify the failure: the runner sets `stalled` for the
|
|
1103
|
+
// inactivity watchdog; here we add `model_error` for failures
|
|
1104
|
+
// attributable to the resolved model (usage limit, auth, quota) so the
|
|
1105
|
+
// parent gets a "switch model" hint instead of a same-model retry
|
|
1106
|
+
// hint, and so canRetryWholeTask skips the pointless same-model
|
|
1107
|
+
// retry.
|
|
1108
|
+
failureKind:
|
|
1109
|
+
r.failureKind ??
|
|
1110
|
+
(r.error && isModelAttributableError(r.error)
|
|
1111
|
+
? "model_error"
|
|
1112
|
+
: undefined),
|
|
1113
|
+
durationMs: r.durationMs,
|
|
1114
|
+
tokens: r.tokens,
|
|
1115
|
+
usage: r.usage,
|
|
1116
|
+
sessionFile,
|
|
1117
|
+
touchedFiles: r.touchedFiles,
|
|
1118
|
+
attributedFiles: r.attributedFiles ?? [],
|
|
831
1119
|
};
|
|
1120
|
+
} finally {
|
|
1121
|
+
// This runs for ordinary success, normal provider failure, whole-task
|
|
1122
|
+
// retry attempts, abort races, stalls, and unexpected throws. Pool hits
|
|
1123
|
+
// remain pool-owned; successful inserts were explicitly released above.
|
|
1124
|
+
if (!sessionReleased) disposeOwnedSession(acquired);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
832
1127
|
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
1128
|
+
function unexpectedAttemptFailure(
|
|
1129
|
+
task: ResolvedTask,
|
|
1130
|
+
err: unknown,
|
|
1131
|
+
accounting: AttemptAccounting,
|
|
1132
|
+
): TaskResult {
|
|
1133
|
+
return {
|
|
1134
|
+
...failTask(task, err instanceof Error ? err.message : String(err)),
|
|
1135
|
+
tokens: accounting.accumulatedUsage.totalTokens,
|
|
1136
|
+
usage: accounting.accumulatedUsage,
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
839
1139
|
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
// drives the prompt and maps events to the progress model. Git failures
|
|
860
|
-
// degrade to an undefined baseline, which tells the runner to skip
|
|
861
|
-
// git-based attribution entirely; see getGitChangedFiles for the
|
|
862
|
-
// contract.
|
|
863
|
-
const gitBaseline = await getGitChangedFiles(task.cwd);
|
|
864
|
-
let r = await runAgentSessionForTesting(
|
|
865
|
-
acquired.session,
|
|
866
|
-
task.prompt,
|
|
867
|
-
{ cwd: task.cwd },
|
|
868
|
-
env.signal,
|
|
869
|
-
onProgress,
|
|
870
|
-
gitBaseline,
|
|
871
|
-
taskStartedAt,
|
|
872
|
-
deadlineAt,
|
|
873
|
-
);
|
|
874
|
-
|
|
875
|
-
cumulativeTokens += r.tokens;
|
|
876
|
-
// The signal can fire after the pre-run check or while the runner is
|
|
877
|
-
// collecting post-prompt evidence. Keep cancellation from looking like
|
|
878
|
-
// success; finally below releases any uncommitted session.
|
|
879
|
-
if (env.signal?.aborted && !r.error) {
|
|
880
|
-
r = { ...r, error: "Aborted" };
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
const sessionFile = resolveResumableSessionFile(
|
|
884
|
-
acquired.sessionFile,
|
|
885
|
-
acquired.sessionManager,
|
|
886
|
-
r.error,
|
|
887
|
-
);
|
|
888
|
-
|
|
889
|
-
if (task.sessionId) {
|
|
890
|
-
if (acquired.lifecycleOwnsSession) {
|
|
891
|
-
// Pool misses (including resumeFrom) transfer ownership only on
|
|
892
|
-
// successful completion; failures are owned by lifecycle and must
|
|
893
|
-
// be disposed in this finally path.
|
|
894
|
-
if (
|
|
895
|
-
!r.error &&
|
|
896
|
-
r.failureKind !== "stalled" &&
|
|
897
|
-
r.failureKind !== "deadline_exceeded"
|
|
898
|
-
) {
|
|
899
|
-
const committed = pool.commit(task.sessionId, {
|
|
900
|
-
session: acquired.session,
|
|
901
|
-
sessionManager: acquired.sessionManager,
|
|
902
|
-
sessionFile: acquired.sessionFile,
|
|
903
|
-
frozen: {
|
|
904
|
-
systemPrompt: task.systemPrompt,
|
|
905
|
-
model: task.model,
|
|
906
|
-
thinking: task.thinking,
|
|
907
|
-
tools: task.tools,
|
|
908
|
-
cwd: task.cwd,
|
|
909
|
-
},
|
|
910
|
-
tokens: r.tokens,
|
|
911
|
-
});
|
|
912
|
-
sessionReleased = sessionReleased || committed;
|
|
913
|
-
}
|
|
914
|
-
} else {
|
|
915
|
-
// A stalled, parent-aborted, or mid-prompt deadline-exceeded pooled
|
|
916
|
-
// attempt is not safe to keep; the session may have been mutated.
|
|
917
|
-
// A pre-prompt deadline (runner never called session.prompt()) left
|
|
918
|
-
// the session in its pre-task state, so return it to the pool intact.
|
|
919
|
-
if (
|
|
920
|
-
r.failureKind === "stalled" ||
|
|
921
|
-
r.error === "Aborted" ||
|
|
922
|
-
(r.failureKind === "deadline_exceeded" && r.prompted !== false)
|
|
923
|
-
) {
|
|
924
|
-
try {
|
|
925
|
-
sessionReleased =
|
|
926
|
-
(await pool._closePooledAgentWithoutLock(task.sessionId)) ||
|
|
927
|
-
sessionReleased;
|
|
928
|
-
} catch (error) {
|
|
929
|
-
// Preserve the primary failure result while logging the cleanup
|
|
930
|
-
// failure explicitly. A pooled session may still be removed by
|
|
931
|
-
// the pool; a pool-miss remains lifecycle-owned and is handled
|
|
932
|
-
// by the finally path above.
|
|
933
|
-
console.error(
|
|
934
|
-
`[delegate] failed to dispose aborted, stalled, or deadline-exceeded pooled session '${task.sessionId}'`,
|
|
935
|
-
error,
|
|
936
|
-
);
|
|
937
|
-
}
|
|
938
|
-
} else if (r.failureKind !== "deadline_exceeded") {
|
|
939
|
-
// Pool hits stay owned by the pool, and non-stalled, non-aborted
|
|
940
|
-
// completions (including failed attempts) must still count usage.
|
|
941
|
-
pool.recordUse(task.sessionId, r.tokens);
|
|
942
|
-
}
|
|
943
|
-
// Pre-prompt deadline (prompted === false): the pooled session was
|
|
944
|
-
// checked out but never used. Leave it in the pool with no usage
|
|
945
|
-
// recorded.
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
accumulatedUsage = addUsage(accumulatedUsage, r.usage);
|
|
950
|
-
cumulativeToolUses += attemptToolUsesObserved;
|
|
951
|
-
|
|
952
|
-
return {
|
|
953
|
-
id: task.id,
|
|
954
|
-
agent: task.agentName,
|
|
955
|
-
output: r.output,
|
|
956
|
-
error: r.error,
|
|
957
|
-
// Classify the failure: the runner sets `stalled` for the
|
|
958
|
-
// inactivity watchdog; here we add `model_error` for failures
|
|
959
|
-
// attributable to the resolved model (usage limit, auth, quota) so the
|
|
960
|
-
// parent gets a "switch model" hint instead of a same-model retry
|
|
961
|
-
// hint, and so canRetryWholeTask skips the pointless same-model
|
|
962
|
-
// retry.
|
|
963
|
-
failureKind:
|
|
964
|
-
r.failureKind ??
|
|
965
|
-
(r.error && isModelAttributableError(r.error)
|
|
966
|
-
? "model_error"
|
|
967
|
-
: undefined),
|
|
968
|
-
durationMs: r.durationMs,
|
|
969
|
-
tokens: r.tokens,
|
|
970
|
-
usage: r.usage,
|
|
971
|
-
sessionFile,
|
|
972
|
-
touchedFiles: r.touchedFiles,
|
|
973
|
-
attributedFiles: r.attributedFiles ?? [],
|
|
974
|
-
};
|
|
975
|
-
} finally {
|
|
976
|
-
// This runs for ordinary success, normal provider failure, whole-task
|
|
977
|
-
// retry attempts, abort races, stalls, and unexpected throws. Pool hits
|
|
978
|
-
// remain pool-owned; successful inserts were explicitly released above.
|
|
979
|
-
if (!sessionReleased) disposeOwnedSession(acquired);
|
|
980
|
-
}
|
|
981
|
-
};
|
|
1140
|
+
/** Merge whole-task accounting into a per-attempt result so no counter can
|
|
1141
|
+
* regress. The single reconcile point for attempt-local values (duration,
|
|
1142
|
+
* tokens, usage) against cumulative totals — used at every exit from the
|
|
1143
|
+
* whole-task retry loop. */
|
|
1144
|
+
function reconcileResultWithAccounting(
|
|
1145
|
+
result: TaskResult,
|
|
1146
|
+
timing: AttemptTiming,
|
|
1147
|
+
accounting: AttemptAccounting,
|
|
1148
|
+
): TaskResult {
|
|
1149
|
+
return {
|
|
1150
|
+
...result,
|
|
1151
|
+
durationMs: Math.max(result.durationMs, Date.now() - timing.taskStartedAt),
|
|
1152
|
+
tokens: Math.max(
|
|
1153
|
+
accounting.cumulativeTokens,
|
|
1154
|
+
accounting.accumulatedUsage.totalTokens,
|
|
1155
|
+
),
|
|
1156
|
+
usage: accounting.accumulatedUsage,
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
982
1159
|
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1160
|
+
/** First attempt plus same-model retries for clearly transient failures. */
|
|
1161
|
+
async function runWithWholeTaskRetries(
|
|
1162
|
+
env: TaskRunEnv,
|
|
1163
|
+
task: ResolvedTask,
|
|
1164
|
+
p: TaskProgress,
|
|
1165
|
+
timing: AttemptTiming,
|
|
1166
|
+
): Promise<TaskOutcome> {
|
|
1167
|
+
const accounting: AttemptAccounting = {
|
|
1168
|
+
hasBashExecution: false,
|
|
1169
|
+
cumulativeTokens: 0,
|
|
1170
|
+
cumulativeToolUses: 0,
|
|
1171
|
+
accumulatedUsage: emptyUsage(),
|
|
1172
|
+
};
|
|
1173
|
+
|
|
1174
|
+
let result: TaskResult;
|
|
1175
|
+
try {
|
|
1176
|
+
result =
|
|
1177
|
+
timing.deadlineAt && Date.now() >= timing.deadlineAt
|
|
1178
|
+
? deadlineExceededResult(task, timing)
|
|
1179
|
+
: await runTaskAttempt(env, task, p, timing, accounting);
|
|
1180
|
+
} catch (err) {
|
|
1181
|
+
// First-attempt throw: finish immediately, no retries.
|
|
1182
|
+
return {
|
|
1183
|
+
result: reconcileResultWithAccounting(
|
|
1184
|
+
unexpectedAttemptFailure(task, err, accounting),
|
|
1185
|
+
timing,
|
|
1186
|
+
accounting,
|
|
1187
|
+
),
|
|
1188
|
+
retries: 0,
|
|
998
1189
|
};
|
|
1190
|
+
}
|
|
999
1191
|
|
|
1000
|
-
|
|
1192
|
+
// An isolated worker is snapshotted once and reconciled once. Retrying the
|
|
1193
|
+
// whole conversation against a mutated proposal tree would make attribution
|
|
1194
|
+
// ambiguous, so v1 deliberately runs one attempt.
|
|
1195
|
+
const maxRetries =
|
|
1196
|
+
task.workspace === "isolated" ? 0 : resolvedWholeTaskMaxRetries(env);
|
|
1197
|
+
const baseDelayMs = resolvedWholeTaskBaseDelayMs(env);
|
|
1198
|
+
let retries = 0;
|
|
1199
|
+
for (
|
|
1200
|
+
let retry = 0;
|
|
1201
|
+
retry < maxRetries &&
|
|
1202
|
+
canRetryWholeTask(task, result, accounting.hasBashExecution);
|
|
1203
|
+
retry++
|
|
1204
|
+
) {
|
|
1205
|
+
const delayMs = baseDelayMs * 2 ** retry;
|
|
1206
|
+
p.durationMs = Math.max(p.durationMs, Date.now() - timing.taskStartedAt);
|
|
1207
|
+
await sleepForWholeTaskRetry(env.signal, delayMs, timing.deadlineAt);
|
|
1208
|
+
if (env.signal?.aborted) {
|
|
1209
|
+
// Preserve any partial output/session path from the last failed attempt
|
|
1210
|
+
// while recording that the retry loop was aborted. The task already
|
|
1211
|
+
// paid for every completed attempt, including the one before sleep;
|
|
1212
|
+
// counters are reconciled by the single return below.
|
|
1213
|
+
result = { ...result, error: "Aborted" };
|
|
1214
|
+
break;
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
if (timing.deadlineAt && Date.now() >= timing.deadlineAt) {
|
|
1218
|
+
result = deadlineExceededResult(task, timing, result);
|
|
1219
|
+
break;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
p.status = "running";
|
|
1223
|
+
p.error = undefined;
|
|
1224
|
+
p.failureKind = undefined;
|
|
1225
|
+
notifyStatusChange(env);
|
|
1226
|
+
retries++;
|
|
1001
1227
|
try {
|
|
1002
|
-
|
|
1003
|
-
result = buildDeadlineExceededResult(undefined);
|
|
1004
|
-
} else {
|
|
1005
|
-
result = await runAttempt();
|
|
1006
|
-
}
|
|
1228
|
+
result = await runTaskAttempt(env, task, p, timing, accounting);
|
|
1007
1229
|
} catch (err) {
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
err instanceof Error ? err.message : String(err),
|
|
1011
|
-
);
|
|
1012
|
-
return finishTask(
|
|
1013
|
-
env,
|
|
1014
|
-
p,
|
|
1015
|
-
{
|
|
1016
|
-
...failure,
|
|
1017
|
-
durationMs: Math.max(failure.durationMs, Date.now() - taskStartedAt),
|
|
1018
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1019
|
-
usage: accumulatedUsage,
|
|
1020
|
-
},
|
|
1021
|
-
task,
|
|
1022
|
-
);
|
|
1230
|
+
result = unexpectedAttemptFailure(task, err, accounting);
|
|
1231
|
+
break;
|
|
1023
1232
|
}
|
|
1233
|
+
}
|
|
1024
1234
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
retry < maxRetries && canRetryWholeTask(task, result, hasBashExecution);
|
|
1031
|
-
retry++
|
|
1032
|
-
) {
|
|
1033
|
-
const delayMs = baseDelayMs * 2 ** retry;
|
|
1034
|
-
p.durationMs = Math.max(p.durationMs, Date.now() - taskStartedAt);
|
|
1035
|
-
await sleepForWholeTaskRetry(env.signal, delayMs, deadlineAt);
|
|
1036
|
-
if (env.signal?.aborted) {
|
|
1037
|
-
// Preserve any partial output/session path from the last failed attempt
|
|
1038
|
-
// while recording that the retry loop was aborted. The task already
|
|
1039
|
-
// paid for every completed attempt, including the one before sleep.
|
|
1040
|
-
result = {
|
|
1041
|
-
...result,
|
|
1042
|
-
error: "Aborted",
|
|
1043
|
-
durationMs: Date.now() - taskStartedAt,
|
|
1044
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1045
|
-
usage: accumulatedUsage,
|
|
1046
|
-
touchedFiles: result.touchedFiles,
|
|
1047
|
-
sessionFile: result.sessionFile,
|
|
1048
|
-
};
|
|
1049
|
-
break;
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
if (deadlineAt && Date.now() >= deadlineAt) {
|
|
1053
|
-
result = buildDeadlineExceededResult(result);
|
|
1054
|
-
break;
|
|
1055
|
-
}
|
|
1235
|
+
return {
|
|
1236
|
+
result: reconcileResultWithAccounting(result, timing, accounting),
|
|
1237
|
+
retries,
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1056
1240
|
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
err instanceof Error ? err.message : String(err),
|
|
1068
|
-
);
|
|
1069
|
-
result = {
|
|
1070
|
-
...failure,
|
|
1071
|
-
durationMs: Math.max(failure.durationMs, Date.now() - taskStartedAt),
|
|
1072
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1073
|
-
usage: accumulatedUsage,
|
|
1074
|
-
};
|
|
1075
|
-
break;
|
|
1076
|
-
}
|
|
1241
|
+
async function runResolvedTaskCore(
|
|
1242
|
+
env: TaskRunEnv,
|
|
1243
|
+
task: ResolvedTask,
|
|
1244
|
+
p: TaskProgress,
|
|
1245
|
+
_taskIndex: number,
|
|
1246
|
+
timing?: AttemptTiming,
|
|
1247
|
+
): Promise<TaskOutcome> {
|
|
1248
|
+
try {
|
|
1249
|
+
if (env.signal?.aborted) {
|
|
1250
|
+
return finishTask(env, p, failTask(task, "Aborted"));
|
|
1077
1251
|
}
|
|
1078
1252
|
|
|
1079
|
-
|
|
1253
|
+
// Primary busy validation is in execute() before ticket creation.
|
|
1254
|
+
// This catches edge cases where validation missed a conflict.
|
|
1255
|
+
const busy = busySessionConflict(env, task);
|
|
1256
|
+
if (busy) return finishTask(env, p, busy);
|
|
1257
|
+
|
|
1258
|
+
p.status = "running";
|
|
1259
|
+
p.model = task.model?.id;
|
|
1260
|
+
|
|
1261
|
+
const sessionActionResult = await applySessionAction(env, task, p);
|
|
1262
|
+
if (sessionActionResult) return sessionActionResult;
|
|
1263
|
+
|
|
1264
|
+
const settled = await runWithWholeTaskRetries(
|
|
1080
1265
|
env,
|
|
1081
|
-
p,
|
|
1082
|
-
{
|
|
1083
|
-
...result,
|
|
1084
|
-
durationMs: Math.max(result.durationMs, Date.now() - taskStartedAt),
|
|
1085
|
-
tokens: Math.max(cumulativeTokens, accumulatedUsage.totalTokens),
|
|
1086
|
-
usage: accumulatedUsage,
|
|
1087
|
-
},
|
|
1088
1266
|
task,
|
|
1089
|
-
|
|
1267
|
+
p,
|
|
1268
|
+
resolveAttemptTiming(task, timing),
|
|
1090
1269
|
);
|
|
1270
|
+
return finishTask(env, p, settled.result, settled.retries);
|
|
1091
1271
|
} catch (err) {
|
|
1092
|
-
// Any acquired session is released by
|
|
1272
|
+
// Any acquired session is released by runTaskAttempt's finally before an
|
|
1093
1273
|
// exception reaches this boundary. This outer catch handles unexpected
|
|
1094
1274
|
// throws before retry accounting is initialized, so report a minimal
|
|
1095
1275
|
// failure without accumulated usage.
|
|
@@ -1097,7 +1277,6 @@ async function runResolvedTaskCore(
|
|
|
1097
1277
|
env,
|
|
1098
1278
|
p,
|
|
1099
1279
|
failTask(task, err instanceof Error ? err.message : String(err)),
|
|
1100
|
-
task,
|
|
1101
1280
|
);
|
|
1102
1281
|
}
|
|
1103
1282
|
}
|