@bermudi/pi-delegate 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -7
- package/agents.ts +140 -14
- package/delegate.ts +3 -2
- package/dispatch.ts +26 -27
- package/extension.ts +2 -4
- package/format.ts +2 -2
- package/host-cache.ts +70 -0
- package/host.ts +164 -811
- package/lifecycle.ts +653 -508
- package/manual.ts +40 -10
- package/package.json +1 -1
- package/pi-package-source.ts +293 -0
- package/provider-extensions.ts +528 -0
- package/quiescence.ts +262 -0
- package/runner.ts +32 -140
- package/schema.ts +77 -153
- package/task-resolution.ts +60 -16
- package/ticket-format.ts +323 -0
- package/tickets.ts +79 -248
- package/tools.ts +12 -0
- package/trusted-paths.ts +71 -0
- package/types.ts +13 -17
package/lifecycle.ts
CHANGED
|
@@ -32,6 +32,9 @@ import { createScratchWorkspace, ScratchDeadlineError } from "./workspace.ts";
|
|
|
32
32
|
/** Internal seam for lifecycle-level tests without replacing session ownership. */
|
|
33
33
|
type RunAgentSession = typeof runAgentSession;
|
|
34
34
|
let runAgentSessionForTesting: RunAgentSession = runAgentSession;
|
|
35
|
+
type CreateScratchWorkspace = typeof createScratchWorkspace;
|
|
36
|
+
let createScratchWorkspaceForTesting: CreateScratchWorkspace =
|
|
37
|
+
createScratchWorkspace;
|
|
35
38
|
|
|
36
39
|
export function _setRunAgentSessionForTesting(
|
|
37
40
|
override: RunAgentSession | undefined,
|
|
@@ -39,6 +42,13 @@ export function _setRunAgentSessionForTesting(
|
|
|
39
42
|
runAgentSessionForTesting = override ?? runAgentSession;
|
|
40
43
|
}
|
|
41
44
|
|
|
45
|
+
/** @internal Test-only scratch materialization seam. */
|
|
46
|
+
export function _setCreateScratchWorkspaceForTesting(
|
|
47
|
+
override: CreateScratchWorkspace | undefined,
|
|
48
|
+
): void {
|
|
49
|
+
createScratchWorkspaceForTesting = override ?? createScratchWorkspace;
|
|
50
|
+
}
|
|
51
|
+
|
|
42
52
|
/**
|
|
43
53
|
* Test-only overrides for whole-task retry settings. When set, these bypass
|
|
44
54
|
* the config-driven values so retry integration tests don't sleep real seconds.
|
|
@@ -152,6 +162,19 @@ interface RunUpdateOffset {
|
|
|
152
162
|
tokensOffset?: number;
|
|
153
163
|
toolUsesOffset?: number;
|
|
154
164
|
}
|
|
165
|
+
/** Advance cumulative progress counters without ever moving backwards —
|
|
166
|
+
* a live TaskProgress row is monotonic across attempts and retries. */
|
|
167
|
+
function bumpProgressCounters(
|
|
168
|
+
p: TaskProgress,
|
|
169
|
+
tokens: number,
|
|
170
|
+
toolUses: number,
|
|
171
|
+
durationMs: number,
|
|
172
|
+
): void {
|
|
173
|
+
p.tokens = Math.max(p.tokens, tokens);
|
|
174
|
+
p.toolUses = Math.max(p.toolUses, toolUses);
|
|
175
|
+
if (durationMs > p.durationMs) p.durationMs = durationMs;
|
|
176
|
+
}
|
|
177
|
+
|
|
155
178
|
/** Mirror a progress update from runAgent into a TaskProgress row.
|
|
156
179
|
*
|
|
157
180
|
* Runner callbacks report attempt-local counters. Offset/merge them so a single
|
|
@@ -162,12 +185,12 @@ export function updateProgressFromRun(
|
|
|
162
185
|
u: AgentProgressUpdate,
|
|
163
186
|
offsets: RunUpdateOffset = {},
|
|
164
187
|
): void {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
188
|
+
bumpProgressCounters(
|
|
189
|
+
p,
|
|
190
|
+
(offsets.tokensOffset ?? 0) + u.tokens,
|
|
191
|
+
(offsets.toolUsesOffset ?? 0) + u.toolUses,
|
|
192
|
+
u.durationMs,
|
|
193
|
+
);
|
|
171
194
|
p.lastActivityAt = u.lastActivityAt;
|
|
172
195
|
p.activities = mergeToolActivities(p.activities, u.activities);
|
|
173
196
|
p.failureKind = u.failureKind;
|
|
@@ -181,38 +204,65 @@ function updateProgressFromResult(p: TaskProgress, r: TaskResult): void {
|
|
|
181
204
|
p.failureKind = r.failureKind;
|
|
182
205
|
}
|
|
183
206
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
*
|
|
187
|
-
|
|
207
|
+
/** Outcome of one logical task run: the final result plus how many same-model
|
|
208
|
+
* retries it took. Telemetry is recorded by the caller at the outermost
|
|
209
|
+
* lifecycle boundary (see recordTaskOutcome). */
|
|
210
|
+
interface TaskOutcome {
|
|
211
|
+
result: TaskResult;
|
|
212
|
+
retries: number;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Notify the optional status observer without making UI/progress delivery part
|
|
216
|
+
* of task correctness. In particular, a host callback must not replace a task
|
|
217
|
+
* result or skip the outer telemetry write. */
|
|
218
|
+
function notifyStatusChange(env: TaskRunEnv): void {
|
|
219
|
+
if (!env.onStatusChange) return;
|
|
220
|
+
try {
|
|
221
|
+
env.onStatusChange();
|
|
222
|
+
} catch (error) {
|
|
223
|
+
console.error("[delegate] task status callback threw; continuing", error);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
188
226
|
|
|
189
227
|
/** Apply a TaskResult to progress and notify the env (sync fires onUpdate).
|
|
190
228
|
* Used at every return point in runResolvedTask — mirrors the old fire() pattern
|
|
191
|
-
* that the duplicated sync/async bodies used after every early-return.
|
|
229
|
+
* that the duplicated sync/async bodies used after every early-return.
|
|
230
|
+
* Telemetry is NOT recorded here: the scratch wrapper may still rewrite the
|
|
231
|
+
* result (path mapping, cleanup errors), so recording happens exactly once,
|
|
232
|
+
* in runResolvedTaskUnlocked, on the final object. */
|
|
192
233
|
function finishTask(
|
|
193
234
|
env: TaskRunEnv,
|
|
194
235
|
p: TaskProgress,
|
|
195
236
|
r: TaskResult,
|
|
196
|
-
task: ResolvedTask,
|
|
197
237
|
retries = 0,
|
|
198
|
-
):
|
|
238
|
+
): TaskOutcome {
|
|
199
239
|
updateProgressFromResult(p, r);
|
|
200
|
-
|
|
240
|
+
notifyStatusChange(env);
|
|
241
|
+
return { result: r, retries };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Record the telemetry task row at the outermost lifecycle boundary — once
|
|
245
|
+
* per runResolvedTask, on the final (post-scratch-wrap) result. Returns the
|
|
246
|
+
* result so call sites stay flat. */
|
|
247
|
+
function recordTaskOutcome(
|
|
248
|
+
env: TaskRunEnv,
|
|
249
|
+
p: TaskProgress,
|
|
250
|
+
task: ResolvedTask,
|
|
251
|
+
outcome: TaskOutcome,
|
|
252
|
+
): TaskResult {
|
|
201
253
|
if (env.telemetryCallId) {
|
|
202
|
-
|
|
254
|
+
recordTask({
|
|
203
255
|
callId: env.telemetryCallId,
|
|
204
256
|
generation: env.telemetryGeneration,
|
|
205
257
|
async: env.async ?? false,
|
|
206
258
|
taskIndex: p.index,
|
|
207
259
|
task,
|
|
208
260
|
progress: p,
|
|
209
|
-
result:
|
|
210
|
-
retries,
|
|
261
|
+
result: outcome.result,
|
|
262
|
+
retries: outcome.retries,
|
|
211
263
|
});
|
|
212
|
-
if (id) taskTelemetryIds.set(r, id);
|
|
213
264
|
}
|
|
214
|
-
|
|
215
|
-
return r;
|
|
265
|
+
return outcome.result;
|
|
216
266
|
}
|
|
217
267
|
|
|
218
268
|
/** A failure attributable to the resolved model/provider — not transient for
|
|
@@ -378,125 +428,122 @@ async function buildDelegateSession(
|
|
|
378
428
|
return session;
|
|
379
429
|
}
|
|
380
430
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
431
|
+
type AcquireResult = AcquiredSession | { error: TaskResult };
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Try a live pooled session. Returns a hit, a formatted mismatch/conflict
|
|
435
|
+
* error, or `undefined` on miss so the caller can resume or create fresh.
|
|
436
|
+
* checkout is pure (no lastUsed bump); lastUsed is bumped by commit().
|
|
437
|
+
*/
|
|
438
|
+
function checkoutPooledSession(
|
|
384
439
|
task: ResolvedTask,
|
|
385
440
|
p: TaskProgress,
|
|
386
|
-
):
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
.map((m) => `${m.field}: '${m.frozen}' vs '${m.requested}'`)
|
|
405
|
-
.join("; ");
|
|
406
|
-
return {
|
|
407
|
-
error: failTask(
|
|
408
|
-
task,
|
|
409
|
-
`Session '${task.sessionId}' config mismatch. Close and recreate: ${detail}`,
|
|
410
|
-
),
|
|
411
|
-
};
|
|
412
|
-
}
|
|
413
|
-
if (co.status === "hit") {
|
|
414
|
-
// A pooled session has its own accumulated context — resumeFrom pointing
|
|
415
|
-
// elsewhere is contradictory. This folds the old defensive agentPool.has
|
|
416
|
-
// precheck: checkout already told us the session is live.
|
|
417
|
-
if (task.resumeFrom) {
|
|
418
|
-
return {
|
|
419
|
-
error: failTask(
|
|
420
|
-
task,
|
|
421
|
-
`resumeFrom conflicts with active sessionId '${task.sessionId}'. The pooled session has its own accumulated context. Close the session first if you want to resume from a different point.`,
|
|
422
|
-
),
|
|
423
|
-
};
|
|
424
|
-
}
|
|
425
|
-
p.model = co.modelId;
|
|
426
|
-
return {
|
|
427
|
-
session: co.session,
|
|
428
|
-
sessionManager: co.sessionManager,
|
|
429
|
-
sessionFile: co.sessionFile,
|
|
430
|
-
lifecycleOwnsSession: false,
|
|
431
|
-
};
|
|
432
|
-
}
|
|
433
|
-
// status === "miss" → fall through to resume / fresh materialization.
|
|
441
|
+
): AcquireResult | undefined {
|
|
442
|
+
if (!task.sessionId) return undefined;
|
|
443
|
+
const co = pool.checkout(task.sessionId, {
|
|
444
|
+
cwd: task.cwd,
|
|
445
|
+
thinking: task.thinking,
|
|
446
|
+
tools: task.tools,
|
|
447
|
+
...task.reuseIntent,
|
|
448
|
+
});
|
|
449
|
+
if (co.status === "mismatch") {
|
|
450
|
+
const detail = co.mismatches
|
|
451
|
+
.map((m) => `${m.field}: '${m.frozen}' vs '${m.requested}'`)
|
|
452
|
+
.join("; ");
|
|
453
|
+
return {
|
|
454
|
+
error: failTask(
|
|
455
|
+
task,
|
|
456
|
+
`Session '${task.sessionId}' config mismatch. Close and recreate: ${detail}`,
|
|
457
|
+
),
|
|
458
|
+
};
|
|
434
459
|
}
|
|
435
|
-
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
// concrete prior conversation we must continue, whereas a sessionId miss just
|
|
439
|
-
// means "create a new pooled session under this id".
|
|
460
|
+
if (co.status !== "hit") return undefined;
|
|
461
|
+
// A pooled session has its own accumulated context — resumeFrom pointing
|
|
462
|
+
// elsewhere is contradictory.
|
|
440
463
|
if (task.resumeFrom) {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
resolvedPath,
|
|
457
|
-
),
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
// Open the existing session and let createAgentSession restore its messages
|
|
461
|
-
// internally (sdk.js reads buildSessionContext().messages + model/thinking).
|
|
462
|
-
let resumed: SessionManager;
|
|
463
|
-
try {
|
|
464
|
-
resumed = SessionManager.open(resolvedPath);
|
|
465
|
-
} catch {
|
|
466
|
-
return {
|
|
467
|
-
error: failTask(
|
|
468
|
-
task,
|
|
469
|
-
`resumeFrom: corrupt session: ${resolvedPath}`,
|
|
470
|
-
resolvedPath,
|
|
471
|
-
),
|
|
472
|
-
};
|
|
473
|
-
}
|
|
474
|
-
// Non-empty sessions have at least the header + the restored branch. An
|
|
475
|
-
// empty/corrupt file surfaces as a session with no restorable messages.
|
|
476
|
-
if (!resumed.buildSessionContext().messages.length) {
|
|
477
|
-
return {
|
|
478
|
-
error: failTask(
|
|
479
|
-
task,
|
|
480
|
-
`resumeFrom: empty session: ${resolvedPath}`,
|
|
481
|
-
resolvedPath,
|
|
482
|
-
),
|
|
483
|
-
};
|
|
484
|
-
}
|
|
464
|
+
return {
|
|
465
|
+
error: failTask(
|
|
466
|
+
task,
|
|
467
|
+
`resumeFrom conflicts with active sessionId '${task.sessionId}'. The pooled session has its own accumulated context. Close the session first if you want to resume from a different point.`,
|
|
468
|
+
),
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
p.model = co.modelId;
|
|
472
|
+
return {
|
|
473
|
+
session: co.session,
|
|
474
|
+
sessionManager: co.sessionManager,
|
|
475
|
+
sessionFile: co.sessionFile,
|
|
476
|
+
lifecycleOwnsSession: false,
|
|
477
|
+
};
|
|
478
|
+
}
|
|
485
479
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
480
|
+
/** Resume takes precedence over a fresh sessionId miss: resumeFrom points at a
|
|
481
|
+
* concrete prior conversation we must continue. */
|
|
482
|
+
async function resumeFromSessionFile(
|
|
483
|
+
env: TaskRunEnv,
|
|
484
|
+
task: ResolvedTask,
|
|
485
|
+
resumeFrom: string,
|
|
486
|
+
): Promise<AcquireResult> {
|
|
487
|
+
const resumeFromPathError = validateResumeFromPath(resumeFrom);
|
|
488
|
+
if (resumeFromPathError) {
|
|
489
|
+
return {
|
|
490
|
+
error: failTask(
|
|
491
|
+
task,
|
|
492
|
+
`resumeFrom: invalid session path: ${resumeFromPathError}; got ${JSON.stringify(resumeFrom)}`,
|
|
493
|
+
),
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
const resolvedPath = resolveCwd(resumeFrom);
|
|
497
|
+
if (!fs.existsSync(resolvedPath)) {
|
|
498
|
+
return {
|
|
499
|
+
error: failTask(
|
|
500
|
+
task,
|
|
501
|
+
`resumeFrom: file not found: ${resolvedPath}`,
|
|
502
|
+
resolvedPath,
|
|
503
|
+
),
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
// Open the existing session and let createAgentSession restore its messages
|
|
507
|
+
// internally (sdk.js reads buildSessionContext().messages + model/thinking).
|
|
508
|
+
let resumed: SessionManager;
|
|
509
|
+
try {
|
|
510
|
+
resumed = SessionManager.open(resolvedPath);
|
|
511
|
+
} catch {
|
|
512
|
+
return {
|
|
513
|
+
error: failTask(
|
|
514
|
+
task,
|
|
515
|
+
`resumeFrom: corrupt session: ${resolvedPath}`,
|
|
516
|
+
resolvedPath,
|
|
517
|
+
),
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
// Non-empty sessions have at least the header + the restored branch. An
|
|
521
|
+
// empty/corrupt file surfaces as a session with no restorable messages.
|
|
522
|
+
if (!resumed.buildSessionContext().messages.length) {
|
|
491
523
|
return {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
524
|
+
error: failTask(
|
|
525
|
+
task,
|
|
526
|
+
`resumeFrom: empty session: ${resolvedPath}`,
|
|
527
|
+
resolvedPath,
|
|
528
|
+
),
|
|
496
529
|
};
|
|
497
530
|
}
|
|
498
531
|
|
|
499
|
-
|
|
532
|
+
const session = await buildDelegateSession(task, resumed, env.modelRegistry);
|
|
533
|
+
return {
|
|
534
|
+
session,
|
|
535
|
+
sessionManager: resumed,
|
|
536
|
+
sessionFile: resolvedPath,
|
|
537
|
+
lifecycleOwnsSession: true,
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
async function createFreshSession(
|
|
542
|
+
env: TaskRunEnv,
|
|
543
|
+
task: ResolvedTask,
|
|
544
|
+
): Promise<AcquireResult> {
|
|
545
|
+
let sessionManager: SessionManager;
|
|
546
|
+
let sessionFile: string | undefined;
|
|
500
547
|
if (task.workspace === "scratch") {
|
|
501
548
|
// A discarded filesystem must not advertise a resumable conversation: a
|
|
502
549
|
// later resume would run against the source cwd and silently lose scratch
|
|
@@ -526,6 +573,20 @@ async function acquireAgentSession(
|
|
|
526
573
|
};
|
|
527
574
|
}
|
|
528
575
|
|
|
576
|
+
/** Resolve the agent + session for a task. Pool hit / resume / fresh. */
|
|
577
|
+
async function acquireAgentSession(
|
|
578
|
+
env: TaskRunEnv,
|
|
579
|
+
task: ResolvedTask,
|
|
580
|
+
p: TaskProgress,
|
|
581
|
+
): Promise<AcquireResult> {
|
|
582
|
+
if (task.sessionId) {
|
|
583
|
+
const pooled = checkoutPooledSession(task, p);
|
|
584
|
+
if (pooled) return pooled;
|
|
585
|
+
}
|
|
586
|
+
if (task.resumeFrom) return resumeFromSessionFile(env, task, task.resumeFrom);
|
|
587
|
+
return createFreshSession(env, task);
|
|
588
|
+
}
|
|
589
|
+
|
|
529
590
|
/**
|
|
530
591
|
* Resolve the `sessionFile` to report on a TaskResult.
|
|
531
592
|
*
|
|
@@ -575,17 +636,22 @@ async function runResolvedTaskUnlocked(
|
|
|
575
636
|
taskIndex: number,
|
|
576
637
|
): Promise<TaskResult> {
|
|
577
638
|
if (task.workspace !== "scratch") {
|
|
578
|
-
|
|
639
|
+
const outcome = await runResolvedTaskCore(env, task, p, taskIndex);
|
|
640
|
+
return recordTaskOutcome(env, p, task, outcome);
|
|
579
641
|
}
|
|
580
642
|
if (task.sessionId || task.resumeFrom || task.sessionAction) {
|
|
581
|
-
return
|
|
643
|
+
return recordTaskOutcome(
|
|
582
644
|
env,
|
|
583
645
|
p,
|
|
584
|
-
failTask(
|
|
585
|
-
task,
|
|
586
|
-
"workspace 'scratch' is one-shot and cannot be combined with sessionId, resumeFrom, or sessionAction.",
|
|
587
|
-
),
|
|
588
646
|
task,
|
|
647
|
+
finishTask(
|
|
648
|
+
env,
|
|
649
|
+
p,
|
|
650
|
+
failTask(
|
|
651
|
+
task,
|
|
652
|
+
"workspace 'scratch' is one-shot and cannot be combined with sessionId, resumeFrom, or sessionAction.",
|
|
653
|
+
),
|
|
654
|
+
),
|
|
589
655
|
);
|
|
590
656
|
}
|
|
591
657
|
|
|
@@ -596,15 +662,20 @@ async function runResolvedTaskUnlocked(
|
|
|
596
662
|
: undefined;
|
|
597
663
|
let workspace: Awaited<ReturnType<typeof createScratchWorkspace>>;
|
|
598
664
|
try {
|
|
599
|
-
workspace = await
|
|
665
|
+
workspace = await createScratchWorkspaceForTesting(
|
|
666
|
+
task.cwd,
|
|
667
|
+
env.signal,
|
|
668
|
+
deadlineAt,
|
|
669
|
+
);
|
|
600
670
|
} catch (error) {
|
|
601
671
|
const setupError = error instanceof Error ? error.message : String(error);
|
|
602
672
|
const deadlineExceeded =
|
|
603
673
|
!env.signal?.aborted && error instanceof ScratchDeadlineError;
|
|
604
|
-
return
|
|
674
|
+
return recordTaskOutcome(
|
|
605
675
|
env,
|
|
606
676
|
p,
|
|
607
|
-
|
|
677
|
+
task,
|
|
678
|
+
finishTask(env, p, {
|
|
608
679
|
...failTask(
|
|
609
680
|
task,
|
|
610
681
|
env.signal?.aborted
|
|
@@ -615,8 +686,7 @@ async function runResolvedTaskUnlocked(
|
|
|
615
686
|
),
|
|
616
687
|
failureKind: deadlineExceeded ? "deadline_exceeded" : undefined,
|
|
617
688
|
durationMs: Date.now() - startedAt,
|
|
618
|
-
},
|
|
619
|
-
task,
|
|
689
|
+
}),
|
|
620
690
|
);
|
|
621
691
|
}
|
|
622
692
|
|
|
@@ -624,27 +694,18 @@ async function runResolvedTaskUnlocked(
|
|
|
624
694
|
...task,
|
|
625
695
|
cwd: workspace.cwd,
|
|
626
696
|
};
|
|
697
|
+
let outcome: TaskOutcome | undefined;
|
|
627
698
|
let result: TaskResult | undefined;
|
|
628
699
|
let cleanupError: string | undefined;
|
|
629
700
|
let needsCorrection = false;
|
|
630
|
-
let telemetryTaskId: string | undefined;
|
|
631
|
-
let retries = 0;
|
|
632
701
|
try {
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
deadlineAt,
|
|
641
|
-
},
|
|
642
|
-
);
|
|
643
|
-
result = preMappingResult;
|
|
644
|
-
// Capture metadata before scratch wrapping creates a new result object.
|
|
645
|
-
// Both values belong to the logical task and must survive that clone.
|
|
646
|
-
telemetryTaskId = taskTelemetryIds.get(result);
|
|
647
|
-
retries = taskRetryCounts.get(result) ?? 0;
|
|
702
|
+
outcome = await runResolvedTaskCore(env, executionTask, p, taskIndex, {
|
|
703
|
+
taskStartedAt: startedAt,
|
|
704
|
+
deadlineAt,
|
|
705
|
+
});
|
|
706
|
+
// Keep the pre-mapping result in `result` so the catch below can preserve
|
|
707
|
+
// its paid-for counters if path mapping throws mid-rewrite.
|
|
708
|
+
result = outcome.result;
|
|
648
709
|
result = {
|
|
649
710
|
...result,
|
|
650
711
|
workspace: "scratch",
|
|
@@ -676,8 +737,8 @@ async function runResolvedTaskUnlocked(
|
|
|
676
737
|
workspace: "scratch",
|
|
677
738
|
durationMs: Date.now() - startedAt,
|
|
678
739
|
};
|
|
679
|
-
// runResolvedTaskCore may already have
|
|
680
|
-
//
|
|
740
|
+
// runResolvedTaskCore may already have notified a successful result
|
|
741
|
+
// before path mapping failed. Correct that observable outcome below.
|
|
681
742
|
needsCorrection = true;
|
|
682
743
|
} finally {
|
|
683
744
|
try {
|
|
@@ -706,390 +767,475 @@ async function runResolvedTaskUnlocked(
|
|
|
706
767
|
durationMs: Math.max(result.durationMs, Date.now() - startedAt),
|
|
707
768
|
};
|
|
708
769
|
updateProgressFromResult(p, result);
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
770
|
+
notifyStatusChange(env);
|
|
771
|
+
}
|
|
772
|
+
// Single telemetry write for the whole logical task — after scratch
|
|
773
|
+
// wrapping and cleanup, on the result actually returned, so no correction
|
|
774
|
+
// upsert is ever needed.
|
|
775
|
+
return recordTaskOutcome(env, p, task, {
|
|
776
|
+
result,
|
|
777
|
+
retries: outcome?.retries ?? 0,
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
interface AttemptTiming {
|
|
782
|
+
taskStartedAt: number;
|
|
783
|
+
deadlineAt: number | undefined;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
interface AttemptAccounting {
|
|
787
|
+
hasBashExecution: boolean;
|
|
788
|
+
cumulativeTokens: number;
|
|
789
|
+
cumulativeToolUses: number;
|
|
790
|
+
accumulatedUsage: ReturnType<typeof emptyUsage>;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
function resolveAttemptTiming(
|
|
794
|
+
task: ResolvedTask,
|
|
795
|
+
timing?: AttemptTiming,
|
|
796
|
+
): AttemptTiming {
|
|
797
|
+
const taskStartedAt = timing?.taskStartedAt ?? Date.now();
|
|
798
|
+
return {
|
|
799
|
+
taskStartedAt,
|
|
800
|
+
deadlineAt:
|
|
801
|
+
timing?.deadlineAt ??
|
|
802
|
+
(task.deadlineMs && task.deadlineMs > 0
|
|
803
|
+
? taskStartedAt + task.deadlineMs
|
|
804
|
+
: undefined),
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/** Close/list short-circuits. Returns undefined when the task should prompt. */
|
|
809
|
+
async function applySessionAction(
|
|
810
|
+
env: TaskRunEnv,
|
|
811
|
+
task: ResolvedTask,
|
|
812
|
+
p: TaskProgress,
|
|
813
|
+
): Promise<TaskOutcome | undefined> {
|
|
814
|
+
if (task.sessionAction === "close") {
|
|
815
|
+
if (!task.sessionId) {
|
|
816
|
+
return finishTask(
|
|
817
|
+
env,
|
|
818
|
+
p,
|
|
819
|
+
failTask(task, "sessionAction='close' requires sessionId."),
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
// The per-session lock for action-based operations is already held by the
|
|
823
|
+
// outer runResolvedTask() wrapper. Use the internal close helper to avoid a
|
|
824
|
+
// reentrant deadlock on the same key.
|
|
825
|
+
const closed = await pool._closePooledAgentWithoutLock(task.sessionId);
|
|
826
|
+
return finishTask(
|
|
827
|
+
env,
|
|
828
|
+
p,
|
|
829
|
+
completeSessionAction(
|
|
719
830
|
task,
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
831
|
+
closed
|
|
832
|
+
? `Session '${task.sessionId}' closed.`
|
|
833
|
+
: `Session '${task.sessionId}' not found.`,
|
|
834
|
+
Date.now() - env.delegateStartedAt,
|
|
835
|
+
),
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
if (task.sessionAction === "list") {
|
|
840
|
+
return finishTask(
|
|
841
|
+
env,
|
|
842
|
+
p,
|
|
843
|
+
completeSessionAction(
|
|
844
|
+
task,
|
|
845
|
+
`Active sessions:\n${pool.listPooledAgents().join("\n")}`,
|
|
846
|
+
Date.now() - env.delegateStartedAt,
|
|
847
|
+
),
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
return undefined;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
function busySessionConflict(
|
|
854
|
+
env: TaskRunEnv,
|
|
855
|
+
task: ResolvedTask,
|
|
856
|
+
): TaskResult | undefined {
|
|
857
|
+
if (!task.sessionId) return undefined;
|
|
858
|
+
const busyTicketId = isSessionBusy(task.sessionId);
|
|
859
|
+
if (busyTicketId && busyTicketId !== env.ticketId) {
|
|
860
|
+
return failTask(
|
|
861
|
+
task,
|
|
862
|
+
`Session '${task.sessionId}' is already in use by ticket ${busyTicketId}. Each session can only handle one task at a time.`,
|
|
863
|
+
);
|
|
864
|
+
}
|
|
865
|
+
return undefined;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
function deadlineExceededResult(
|
|
869
|
+
task: ResolvedTask,
|
|
870
|
+
timing: AttemptTiming,
|
|
871
|
+
prior?: TaskResult,
|
|
872
|
+
): TaskResult {
|
|
873
|
+
const budgetMs = Math.max(0, (timing.deadlineAt ?? 0) - timing.taskStartedAt);
|
|
874
|
+
return {
|
|
875
|
+
id: task.id,
|
|
876
|
+
agent: task.agentName,
|
|
877
|
+
output: prior?.output ?? "",
|
|
878
|
+
error: formatDeadlineExceededError(budgetMs),
|
|
879
|
+
failureKind: "deadline_exceeded",
|
|
880
|
+
durationMs: prior?.durationMs ?? 0,
|
|
881
|
+
tokens: prior?.tokens ?? 0,
|
|
882
|
+
usage: prior?.usage ?? emptyUsage(),
|
|
883
|
+
sessionFile: prior?.sessionFile,
|
|
884
|
+
touchedFiles: prior?.touchedFiles ?? [],
|
|
885
|
+
attributedFiles: prior?.attributedFiles ?? [],
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
function noteAttemptProgress(
|
|
890
|
+
env: TaskRunEnv,
|
|
891
|
+
p: TaskProgress,
|
|
892
|
+
u: AgentProgressUpdate,
|
|
893
|
+
timing: AttemptTiming,
|
|
894
|
+
accounting: AttemptAccounting,
|
|
895
|
+
): void {
|
|
896
|
+
if (
|
|
897
|
+
!accounting.hasBashExecution &&
|
|
898
|
+
u.activities.some((activity) => activity.name === "bash")
|
|
899
|
+
) {
|
|
900
|
+
accounting.hasBashExecution = true;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
const mapped: AgentProgressUpdate = {
|
|
904
|
+
...u,
|
|
905
|
+
tokens: accounting.cumulativeTokens + u.tokens,
|
|
906
|
+
toolUses: accounting.cumulativeToolUses + u.toolUses,
|
|
907
|
+
durationMs: Date.now() - timing.taskStartedAt,
|
|
908
|
+
};
|
|
909
|
+
|
|
910
|
+
// Keep live totals monotonic across attempts.
|
|
911
|
+
bumpProgressCounters(p, mapped.tokens, mapped.toolUses, mapped.durationMs);
|
|
912
|
+
env.onProgress(p, mapped);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/** Commit, record, or evict a pooled session after one prompt attempt. */
|
|
916
|
+
async function settlePooledAttempt(
|
|
917
|
+
task: ResolvedTask,
|
|
918
|
+
acquired: AcquiredSession,
|
|
919
|
+
r: {
|
|
920
|
+
error?: string;
|
|
921
|
+
failureKind?: TaskResult["failureKind"];
|
|
922
|
+
prompted?: boolean;
|
|
923
|
+
tokens: number;
|
|
924
|
+
},
|
|
925
|
+
sessionReleased: boolean,
|
|
926
|
+
): Promise<boolean> {
|
|
927
|
+
if (!task.sessionId) return sessionReleased;
|
|
928
|
+
if (acquired.lifecycleOwnsSession) {
|
|
929
|
+
// Pool misses (including resumeFrom) transfer ownership only on
|
|
930
|
+
// successful completion; failures are owned by lifecycle and must
|
|
931
|
+
// be disposed in this finally path.
|
|
932
|
+
if (
|
|
933
|
+
!r.error &&
|
|
934
|
+
r.failureKind !== "stalled" &&
|
|
935
|
+
r.failureKind !== "deadline_exceeded"
|
|
936
|
+
) {
|
|
937
|
+
const committed = pool.commit(task.sessionId, {
|
|
938
|
+
session: acquired.session,
|
|
939
|
+
sessionManager: acquired.sessionManager,
|
|
940
|
+
sessionFile: acquired.sessionFile,
|
|
941
|
+
frozen: {
|
|
942
|
+
systemPrompt: task.systemPrompt,
|
|
943
|
+
model: task.model,
|
|
944
|
+
thinking: task.thinking,
|
|
945
|
+
tools: task.tools,
|
|
946
|
+
cwd: task.cwd,
|
|
947
|
+
},
|
|
948
|
+
tokens: r.tokens,
|
|
723
949
|
});
|
|
950
|
+
return sessionReleased || committed;
|
|
951
|
+
}
|
|
952
|
+
return sessionReleased;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
// A stalled, parent-aborted, or mid-prompt deadline-exceeded pooled
|
|
956
|
+
// attempt is not safe to keep; the session may have been mutated.
|
|
957
|
+
// A pre-prompt deadline (runner never called session.prompt()) left
|
|
958
|
+
// the session in its pre-task state, so return it to the pool intact.
|
|
959
|
+
if (
|
|
960
|
+
r.failureKind === "stalled" ||
|
|
961
|
+
r.error === "Aborted" ||
|
|
962
|
+
(r.failureKind === "deadline_exceeded" && r.prompted !== false)
|
|
963
|
+
) {
|
|
964
|
+
try {
|
|
965
|
+
return (
|
|
966
|
+
(await pool._closePooledAgentWithoutLock(task.sessionId)) ||
|
|
967
|
+
sessionReleased
|
|
968
|
+
);
|
|
969
|
+
} catch (error) {
|
|
970
|
+
// Preserve the primary failure result while logging the cleanup
|
|
971
|
+
// failure explicitly. A pooled session may still be removed by
|
|
972
|
+
// the pool; a pool-miss remains lifecycle-owned and is handled
|
|
973
|
+
// by the finally path above.
|
|
974
|
+
console.error(
|
|
975
|
+
`[delegate] failed to dispose aborted, stalled, or deadline-exceeded pooled session '${task.sessionId}'`,
|
|
976
|
+
error,
|
|
977
|
+
);
|
|
978
|
+
return sessionReleased;
|
|
724
979
|
}
|
|
725
|
-
env.onStatusChange?.();
|
|
726
980
|
}
|
|
727
|
-
|
|
981
|
+
if (r.failureKind !== "deadline_exceeded") {
|
|
982
|
+
// Pool hits stay owned by the pool, and non-stalled, non-aborted
|
|
983
|
+
// completions (including failed attempts) must still count usage.
|
|
984
|
+
pool.recordUse(task.sessionId, r.tokens);
|
|
985
|
+
}
|
|
986
|
+
// Pre-prompt deadline (prompted === false): the pooled session was
|
|
987
|
+
// checked out but never used. Leave it in the pool with no usage
|
|
988
|
+
// recorded.
|
|
989
|
+
return sessionReleased;
|
|
728
990
|
}
|
|
729
991
|
|
|
730
|
-
|
|
992
|
+
/** Acquire, prompt, and settle one attempt. Mutates `accounting` on success. */
|
|
993
|
+
async function runTaskAttempt(
|
|
731
994
|
env: TaskRunEnv,
|
|
732
995
|
task: ResolvedTask,
|
|
733
996
|
p: TaskProgress,
|
|
734
|
-
|
|
735
|
-
|
|
997
|
+
timing: AttemptTiming,
|
|
998
|
+
accounting: AttemptAccounting,
|
|
736
999
|
): Promise<TaskResult> {
|
|
1000
|
+
let attemptToolUsesObserved = 0;
|
|
1001
|
+
const onProgress = (u: AgentProgressUpdate): void => {
|
|
1002
|
+
attemptToolUsesObserved = Math.max(attemptToolUsesObserved, u.toolUses);
|
|
1003
|
+
noteAttemptProgress(env, p, u, timing, accounting);
|
|
1004
|
+
};
|
|
1005
|
+
|
|
1006
|
+
const acquired = await acquireAgentSession(env, task, p);
|
|
1007
|
+
if ("error" in acquired) return acquired.error;
|
|
1008
|
+
|
|
1009
|
+
// A pool hit is already owned by the pool. Fresh/resumed sessions belong
|
|
1010
|
+
// to this attempt until commit/recordUse/close logic runs.
|
|
1011
|
+
let sessionReleased = !acquired.lifecycleOwnsSession;
|
|
737
1012
|
try {
|
|
738
|
-
//
|
|
1013
|
+
// Re-check abort after acquisition. The pre-acquire check at the top can
|
|
1014
|
+
// miss a signal that fires during getHostDeps/createAgentSession/git
|
|
1015
|
+
// baseline. runAgentSession re-checks after attaching its listener, but a
|
|
1016
|
+
// cancelled ticket should not even start the subagent (no file writes, no
|
|
1017
|
+
// pool insert).
|
|
739
1018
|
if (env.signal?.aborted) {
|
|
740
|
-
return
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
// ── Session busy guard (defense-in-depth) ────────────────────────
|
|
744
|
-
// Primary validation is in execute() before ticket creation.
|
|
745
|
-
// This catches edge cases where validation missed a conflict.
|
|
746
|
-
if (task.sessionId) {
|
|
747
|
-
const busyTicketId = isSessionBusy(task.sessionId);
|
|
748
|
-
if (busyTicketId && busyTicketId !== env.ticketId) {
|
|
749
|
-
const msg = `Session '${task.sessionId}' is already in use by ticket ${busyTicketId}. Each session can only handle one task at a time.`;
|
|
750
|
-
return finishTask(env, p, failTask(task, msg), task);
|
|
751
|
-
}
|
|
1019
|
+
return failTask(task, "Aborted");
|
|
752
1020
|
}
|
|
753
1021
|
|
|
754
|
-
|
|
755
|
-
|
|
1022
|
+
// Snapshot git status before the run so touchedFiles can diff after.
|
|
1023
|
+
// AgentSession owns retry/compaction internally — runAgentSession just
|
|
1024
|
+
// drives the prompt and maps events to the progress model. Git failures
|
|
1025
|
+
// degrade to an undefined baseline, which tells the runner to skip
|
|
1026
|
+
// git-based attribution entirely; see getGitChangedFiles for the
|
|
1027
|
+
// contract.
|
|
1028
|
+
const gitBaseline = await getGitChangedFiles(task.cwd);
|
|
1029
|
+
let r = await runAgentSessionForTesting(
|
|
1030
|
+
acquired.session,
|
|
1031
|
+
task.prompt,
|
|
1032
|
+
{ cwd: task.cwd },
|
|
1033
|
+
env.signal,
|
|
1034
|
+
onProgress,
|
|
1035
|
+
gitBaseline,
|
|
1036
|
+
timing.taskStartedAt,
|
|
1037
|
+
timing.deadlineAt,
|
|
1038
|
+
);
|
|
756
1039
|
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
failTask(task, "sessionAction='close' requires sessionId."),
|
|
764
|
-
task,
|
|
765
|
-
);
|
|
766
|
-
}
|
|
767
|
-
// The per-session lock for action-based operations is already held by the
|
|
768
|
-
// outer runResolvedTask() wrapper. Use the internal close helper to avoid a
|
|
769
|
-
// reentrant deadlock on the same key.
|
|
770
|
-
const closed = await pool._closePooledAgentWithoutLock(task.sessionId);
|
|
771
|
-
return finishTask(
|
|
772
|
-
env,
|
|
773
|
-
p,
|
|
774
|
-
completeSessionAction(
|
|
775
|
-
task,
|
|
776
|
-
closed
|
|
777
|
-
? `Session '${task.sessionId}' closed.`
|
|
778
|
-
: `Session '${task.sessionId}' not found.`,
|
|
779
|
-
Date.now() - env.delegateStartedAt,
|
|
780
|
-
),
|
|
781
|
-
task,
|
|
782
|
-
);
|
|
1040
|
+
accounting.cumulativeTokens += r.tokens;
|
|
1041
|
+
// The signal can fire after the pre-run check or while the runner is
|
|
1042
|
+
// collecting post-prompt evidence. Keep cancellation from looking like
|
|
1043
|
+
// success; finally below releases any uncommitted session.
|
|
1044
|
+
if (env.signal?.aborted && !r.error) {
|
|
1045
|
+
r = { ...r, error: "Aborted" };
|
|
783
1046
|
}
|
|
784
1047
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
task,
|
|
791
|
-
`Active sessions:\n${pool.listPooledAgents().join("\n")}`,
|
|
792
|
-
Date.now() - env.delegateStartedAt,
|
|
793
|
-
),
|
|
794
|
-
task,
|
|
795
|
-
);
|
|
796
|
-
}
|
|
1048
|
+
const sessionFile = resolveResumableSessionFile(
|
|
1049
|
+
acquired.sessionFile,
|
|
1050
|
+
acquired.sessionManager,
|
|
1051
|
+
r.error,
|
|
1052
|
+
);
|
|
797
1053
|
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
(task.deadlineMs && task.deadlineMs > 0
|
|
805
|
-
? taskStartedAt + task.deadlineMs
|
|
806
|
-
: undefined);
|
|
807
|
-
let accumulatedUsage = emptyUsage();
|
|
808
|
-
|
|
809
|
-
const onAttemptProgress = (u: AgentProgressUpdate): void => {
|
|
810
|
-
if (
|
|
811
|
-
!hasBashExecution &&
|
|
812
|
-
u.activities.some((activity) => activity.name === "bash")
|
|
813
|
-
) {
|
|
814
|
-
hasBashExecution = true;
|
|
815
|
-
}
|
|
1054
|
+
sessionReleased = await settlePooledAttempt(
|
|
1055
|
+
task,
|
|
1056
|
+
acquired,
|
|
1057
|
+
r,
|
|
1058
|
+
sessionReleased,
|
|
1059
|
+
);
|
|
816
1060
|
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
};
|
|
1061
|
+
accounting.accumulatedUsage = addUsage(
|
|
1062
|
+
accounting.accumulatedUsage,
|
|
1063
|
+
r.usage,
|
|
1064
|
+
);
|
|
1065
|
+
accounting.cumulativeToolUses += attemptToolUsesObserved;
|
|
823
1066
|
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
1067
|
+
return {
|
|
1068
|
+
id: task.id,
|
|
1069
|
+
agent: task.agentName,
|
|
1070
|
+
output: r.output,
|
|
1071
|
+
error: r.error,
|
|
1072
|
+
// Classify the failure: the runner sets `stalled` for the
|
|
1073
|
+
// inactivity watchdog; here we add `model_error` for failures
|
|
1074
|
+
// attributable to the resolved model (usage limit, auth, quota) so the
|
|
1075
|
+
// parent gets a "switch model" hint instead of a same-model retry
|
|
1076
|
+
// hint, and so canRetryWholeTask skips the pointless same-model
|
|
1077
|
+
// retry.
|
|
1078
|
+
failureKind:
|
|
1079
|
+
r.failureKind ??
|
|
1080
|
+
(r.error && isModelAttributableError(r.error)
|
|
1081
|
+
? "model_error"
|
|
1082
|
+
: undefined),
|
|
1083
|
+
durationMs: r.durationMs,
|
|
1084
|
+
tokens: r.tokens,
|
|
1085
|
+
usage: r.usage,
|
|
1086
|
+
sessionFile,
|
|
1087
|
+
touchedFiles: r.touchedFiles,
|
|
1088
|
+
attributedFiles: r.attributedFiles ?? [],
|
|
831
1089
|
};
|
|
1090
|
+
} finally {
|
|
1091
|
+
// This runs for ordinary success, normal provider failure, whole-task
|
|
1092
|
+
// retry attempts, abort races, stalls, and unexpected throws. Pool hits
|
|
1093
|
+
// remain pool-owned; successful inserts were explicitly released above.
|
|
1094
|
+
if (!sessionReleased) disposeOwnedSession(acquired);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
832
1097
|
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
1098
|
+
function unexpectedAttemptFailure(
|
|
1099
|
+
task: ResolvedTask,
|
|
1100
|
+
err: unknown,
|
|
1101
|
+
accounting: AttemptAccounting,
|
|
1102
|
+
): TaskResult {
|
|
1103
|
+
return {
|
|
1104
|
+
...failTask(task, err instanceof Error ? err.message : String(err)),
|
|
1105
|
+
tokens: accounting.accumulatedUsage.totalTokens,
|
|
1106
|
+
usage: accounting.accumulatedUsage,
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
839
1109
|
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
// drives the prompt and maps events to the progress model. Git failures
|
|
860
|
-
// degrade to an undefined baseline, which tells the runner to skip
|
|
861
|
-
// git-based attribution entirely; see getGitChangedFiles for the
|
|
862
|
-
// contract.
|
|
863
|
-
const gitBaseline = await getGitChangedFiles(task.cwd);
|
|
864
|
-
let r = await runAgentSessionForTesting(
|
|
865
|
-
acquired.session,
|
|
866
|
-
task.prompt,
|
|
867
|
-
{ cwd: task.cwd },
|
|
868
|
-
env.signal,
|
|
869
|
-
onProgress,
|
|
870
|
-
gitBaseline,
|
|
871
|
-
taskStartedAt,
|
|
872
|
-
deadlineAt,
|
|
873
|
-
);
|
|
874
|
-
|
|
875
|
-
cumulativeTokens += r.tokens;
|
|
876
|
-
// The signal can fire after the pre-run check or while the runner is
|
|
877
|
-
// collecting post-prompt evidence. Keep cancellation from looking like
|
|
878
|
-
// success; finally below releases any uncommitted session.
|
|
879
|
-
if (env.signal?.aborted && !r.error) {
|
|
880
|
-
r = { ...r, error: "Aborted" };
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
const sessionFile = resolveResumableSessionFile(
|
|
884
|
-
acquired.sessionFile,
|
|
885
|
-
acquired.sessionManager,
|
|
886
|
-
r.error,
|
|
887
|
-
);
|
|
888
|
-
|
|
889
|
-
if (task.sessionId) {
|
|
890
|
-
if (acquired.lifecycleOwnsSession) {
|
|
891
|
-
// Pool misses (including resumeFrom) transfer ownership only on
|
|
892
|
-
// successful completion; failures are owned by lifecycle and must
|
|
893
|
-
// be disposed in this finally path.
|
|
894
|
-
if (
|
|
895
|
-
!r.error &&
|
|
896
|
-
r.failureKind !== "stalled" &&
|
|
897
|
-
r.failureKind !== "deadline_exceeded"
|
|
898
|
-
) {
|
|
899
|
-
const committed = pool.commit(task.sessionId, {
|
|
900
|
-
session: acquired.session,
|
|
901
|
-
sessionManager: acquired.sessionManager,
|
|
902
|
-
sessionFile: acquired.sessionFile,
|
|
903
|
-
frozen: {
|
|
904
|
-
systemPrompt: task.systemPrompt,
|
|
905
|
-
model: task.model,
|
|
906
|
-
thinking: task.thinking,
|
|
907
|
-
tools: task.tools,
|
|
908
|
-
cwd: task.cwd,
|
|
909
|
-
},
|
|
910
|
-
tokens: r.tokens,
|
|
911
|
-
});
|
|
912
|
-
sessionReleased = sessionReleased || committed;
|
|
913
|
-
}
|
|
914
|
-
} else {
|
|
915
|
-
// A stalled, parent-aborted, or mid-prompt deadline-exceeded pooled
|
|
916
|
-
// attempt is not safe to keep; the session may have been mutated.
|
|
917
|
-
// A pre-prompt deadline (runner never called session.prompt()) left
|
|
918
|
-
// the session in its pre-task state, so return it to the pool intact.
|
|
919
|
-
if (
|
|
920
|
-
r.failureKind === "stalled" ||
|
|
921
|
-
r.error === "Aborted" ||
|
|
922
|
-
(r.failureKind === "deadline_exceeded" && r.prompted !== false)
|
|
923
|
-
) {
|
|
924
|
-
try {
|
|
925
|
-
sessionReleased =
|
|
926
|
-
(await pool._closePooledAgentWithoutLock(task.sessionId)) ||
|
|
927
|
-
sessionReleased;
|
|
928
|
-
} catch (error) {
|
|
929
|
-
// Preserve the primary failure result while logging the cleanup
|
|
930
|
-
// failure explicitly. A pooled session may still be removed by
|
|
931
|
-
// the pool; a pool-miss remains lifecycle-owned and is handled
|
|
932
|
-
// by the finally path above.
|
|
933
|
-
console.error(
|
|
934
|
-
`[delegate] failed to dispose aborted, stalled, or deadline-exceeded pooled session '${task.sessionId}'`,
|
|
935
|
-
error,
|
|
936
|
-
);
|
|
937
|
-
}
|
|
938
|
-
} else if (r.failureKind !== "deadline_exceeded") {
|
|
939
|
-
// Pool hits stay owned by the pool, and non-stalled, non-aborted
|
|
940
|
-
// completions (including failed attempts) must still count usage.
|
|
941
|
-
pool.recordUse(task.sessionId, r.tokens);
|
|
942
|
-
}
|
|
943
|
-
// Pre-prompt deadline (prompted === false): the pooled session was
|
|
944
|
-
// checked out but never used. Leave it in the pool with no usage
|
|
945
|
-
// recorded.
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
accumulatedUsage = addUsage(accumulatedUsage, r.usage);
|
|
950
|
-
cumulativeToolUses += attemptToolUsesObserved;
|
|
951
|
-
|
|
952
|
-
return {
|
|
953
|
-
id: task.id,
|
|
954
|
-
agent: task.agentName,
|
|
955
|
-
output: r.output,
|
|
956
|
-
error: r.error,
|
|
957
|
-
// Classify the failure: the runner sets `stalled` for the
|
|
958
|
-
// inactivity watchdog; here we add `model_error` for failures
|
|
959
|
-
// attributable to the resolved model (usage limit, auth, quota) so the
|
|
960
|
-
// parent gets a "switch model" hint instead of a same-model retry
|
|
961
|
-
// hint, and so canRetryWholeTask skips the pointless same-model
|
|
962
|
-
// retry.
|
|
963
|
-
failureKind:
|
|
964
|
-
r.failureKind ??
|
|
965
|
-
(r.error && isModelAttributableError(r.error)
|
|
966
|
-
? "model_error"
|
|
967
|
-
: undefined),
|
|
968
|
-
durationMs: r.durationMs,
|
|
969
|
-
tokens: r.tokens,
|
|
970
|
-
usage: r.usage,
|
|
971
|
-
sessionFile,
|
|
972
|
-
touchedFiles: r.touchedFiles,
|
|
973
|
-
attributedFiles: r.attributedFiles ?? [],
|
|
974
|
-
};
|
|
975
|
-
} finally {
|
|
976
|
-
// This runs for ordinary success, normal provider failure, whole-task
|
|
977
|
-
// retry attempts, abort races, stalls, and unexpected throws. Pool hits
|
|
978
|
-
// remain pool-owned; successful inserts were explicitly released above.
|
|
979
|
-
if (!sessionReleased) disposeOwnedSession(acquired);
|
|
980
|
-
}
|
|
981
|
-
};
|
|
1110
|
+
/** Merge whole-task accounting into a per-attempt result so no counter can
|
|
1111
|
+
* regress. The single reconcile point for attempt-local values (duration,
|
|
1112
|
+
* tokens, usage) against cumulative totals — used at every exit from the
|
|
1113
|
+
* whole-task retry loop. */
|
|
1114
|
+
function reconcileResultWithAccounting(
|
|
1115
|
+
result: TaskResult,
|
|
1116
|
+
timing: AttemptTiming,
|
|
1117
|
+
accounting: AttemptAccounting,
|
|
1118
|
+
): TaskResult {
|
|
1119
|
+
return {
|
|
1120
|
+
...result,
|
|
1121
|
+
durationMs: Math.max(result.durationMs, Date.now() - timing.taskStartedAt),
|
|
1122
|
+
tokens: Math.max(
|
|
1123
|
+
accounting.cumulativeTokens,
|
|
1124
|
+
accounting.accumulatedUsage.totalTokens,
|
|
1125
|
+
),
|
|
1126
|
+
usage: accounting.accumulatedUsage,
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
982
1129
|
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1130
|
+
/** First attempt plus same-model retries for clearly transient failures. */
|
|
1131
|
+
async function runWithWholeTaskRetries(
|
|
1132
|
+
env: TaskRunEnv,
|
|
1133
|
+
task: ResolvedTask,
|
|
1134
|
+
p: TaskProgress,
|
|
1135
|
+
timing: AttemptTiming,
|
|
1136
|
+
): Promise<TaskOutcome> {
|
|
1137
|
+
const accounting: AttemptAccounting = {
|
|
1138
|
+
hasBashExecution: false,
|
|
1139
|
+
cumulativeTokens: 0,
|
|
1140
|
+
cumulativeToolUses: 0,
|
|
1141
|
+
accumulatedUsage: emptyUsage(),
|
|
1142
|
+
};
|
|
1143
|
+
|
|
1144
|
+
let result: TaskResult;
|
|
1145
|
+
try {
|
|
1146
|
+
result =
|
|
1147
|
+
timing.deadlineAt && Date.now() >= timing.deadlineAt
|
|
1148
|
+
? deadlineExceededResult(task, timing)
|
|
1149
|
+
: await runTaskAttempt(env, task, p, timing, accounting);
|
|
1150
|
+
} catch (err) {
|
|
1151
|
+
// First-attempt throw: finish immediately, no retries.
|
|
1152
|
+
return {
|
|
1153
|
+
result: reconcileResultWithAccounting(
|
|
1154
|
+
unexpectedAttemptFailure(task, err, accounting),
|
|
1155
|
+
timing,
|
|
1156
|
+
accounting,
|
|
1157
|
+
),
|
|
1158
|
+
retries: 0,
|
|
998
1159
|
};
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
const maxRetries = resolvedWholeTaskMaxRetries();
|
|
1163
|
+
const baseDelayMs = resolvedWholeTaskBaseDelayMs();
|
|
1164
|
+
let retries = 0;
|
|
1165
|
+
for (
|
|
1166
|
+
let retry = 0;
|
|
1167
|
+
retry < maxRetries &&
|
|
1168
|
+
canRetryWholeTask(task, result, accounting.hasBashExecution);
|
|
1169
|
+
retry++
|
|
1170
|
+
) {
|
|
1171
|
+
const delayMs = baseDelayMs * 2 ** retry;
|
|
1172
|
+
p.durationMs = Math.max(p.durationMs, Date.now() - timing.taskStartedAt);
|
|
1173
|
+
await sleepForWholeTaskRetry(env.signal, delayMs, timing.deadlineAt);
|
|
1174
|
+
if (env.signal?.aborted) {
|
|
1175
|
+
// Preserve any partial output/session path from the last failed attempt
|
|
1176
|
+
// while recording that the retry loop was aborted. The task already
|
|
1177
|
+
// paid for every completed attempt, including the one before sleep;
|
|
1178
|
+
// counters are reconciled by the single return below.
|
|
1179
|
+
result = { ...result, error: "Aborted" };
|
|
1180
|
+
break;
|
|
1181
|
+
}
|
|
999
1182
|
|
|
1000
|
-
|
|
1183
|
+
if (timing.deadlineAt && Date.now() >= timing.deadlineAt) {
|
|
1184
|
+
result = deadlineExceededResult(task, timing, result);
|
|
1185
|
+
break;
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
p.status = "running";
|
|
1189
|
+
p.error = undefined;
|
|
1190
|
+
p.failureKind = undefined;
|
|
1191
|
+
notifyStatusChange(env);
|
|
1192
|
+
retries++;
|
|
1001
1193
|
try {
|
|
1002
|
-
|
|
1003
|
-
result = buildDeadlineExceededResult(undefined);
|
|
1004
|
-
} else {
|
|
1005
|
-
result = await runAttempt();
|
|
1006
|
-
}
|
|
1194
|
+
result = await runTaskAttempt(env, task, p, timing, accounting);
|
|
1007
1195
|
} catch (err) {
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
err instanceof Error ? err.message : String(err),
|
|
1011
|
-
);
|
|
1012
|
-
return finishTask(
|
|
1013
|
-
env,
|
|
1014
|
-
p,
|
|
1015
|
-
{
|
|
1016
|
-
...failure,
|
|
1017
|
-
durationMs: Math.max(failure.durationMs, Date.now() - taskStartedAt),
|
|
1018
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1019
|
-
usage: accumulatedUsage,
|
|
1020
|
-
},
|
|
1021
|
-
task,
|
|
1022
|
-
);
|
|
1196
|
+
result = unexpectedAttemptFailure(task, err, accounting);
|
|
1197
|
+
break;
|
|
1023
1198
|
}
|
|
1199
|
+
}
|
|
1024
1200
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
retry < maxRetries && canRetryWholeTask(task, result, hasBashExecution);
|
|
1031
|
-
retry++
|
|
1032
|
-
) {
|
|
1033
|
-
const delayMs = baseDelayMs * 2 ** retry;
|
|
1034
|
-
p.durationMs = Math.max(p.durationMs, Date.now() - taskStartedAt);
|
|
1035
|
-
await sleepForWholeTaskRetry(env.signal, delayMs, deadlineAt);
|
|
1036
|
-
if (env.signal?.aborted) {
|
|
1037
|
-
// Preserve any partial output/session path from the last failed attempt
|
|
1038
|
-
// while recording that the retry loop was aborted. The task already
|
|
1039
|
-
// paid for every completed attempt, including the one before sleep.
|
|
1040
|
-
result = {
|
|
1041
|
-
...result,
|
|
1042
|
-
error: "Aborted",
|
|
1043
|
-
durationMs: Date.now() - taskStartedAt,
|
|
1044
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1045
|
-
usage: accumulatedUsage,
|
|
1046
|
-
touchedFiles: result.touchedFiles,
|
|
1047
|
-
sessionFile: result.sessionFile,
|
|
1048
|
-
};
|
|
1049
|
-
break;
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
if (deadlineAt && Date.now() >= deadlineAt) {
|
|
1053
|
-
result = buildDeadlineExceededResult(result);
|
|
1054
|
-
break;
|
|
1055
|
-
}
|
|
1201
|
+
return {
|
|
1202
|
+
result: reconcileResultWithAccounting(result, timing, accounting),
|
|
1203
|
+
retries,
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1056
1206
|
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
err instanceof Error ? err.message : String(err),
|
|
1068
|
-
);
|
|
1069
|
-
result = {
|
|
1070
|
-
...failure,
|
|
1071
|
-
durationMs: Math.max(failure.durationMs, Date.now() - taskStartedAt),
|
|
1072
|
-
tokens: accumulatedUsage.totalTokens,
|
|
1073
|
-
usage: accumulatedUsage,
|
|
1074
|
-
};
|
|
1075
|
-
break;
|
|
1076
|
-
}
|
|
1207
|
+
async function runResolvedTaskCore(
|
|
1208
|
+
env: TaskRunEnv,
|
|
1209
|
+
task: ResolvedTask,
|
|
1210
|
+
p: TaskProgress,
|
|
1211
|
+
_taskIndex: number,
|
|
1212
|
+
timing?: AttemptTiming,
|
|
1213
|
+
): Promise<TaskOutcome> {
|
|
1214
|
+
try {
|
|
1215
|
+
if (env.signal?.aborted) {
|
|
1216
|
+
return finishTask(env, p, failTask(task, "Aborted"));
|
|
1077
1217
|
}
|
|
1078
1218
|
|
|
1079
|
-
|
|
1219
|
+
// Primary busy validation is in execute() before ticket creation.
|
|
1220
|
+
// This catches edge cases where validation missed a conflict.
|
|
1221
|
+
const busy = busySessionConflict(env, task);
|
|
1222
|
+
if (busy) return finishTask(env, p, busy);
|
|
1223
|
+
|
|
1224
|
+
p.status = "running";
|
|
1225
|
+
p.model = task.model?.id;
|
|
1226
|
+
|
|
1227
|
+
const sessionActionResult = await applySessionAction(env, task, p);
|
|
1228
|
+
if (sessionActionResult) return sessionActionResult;
|
|
1229
|
+
|
|
1230
|
+
const settled = await runWithWholeTaskRetries(
|
|
1080
1231
|
env,
|
|
1081
|
-
p,
|
|
1082
|
-
{
|
|
1083
|
-
...result,
|
|
1084
|
-
durationMs: Math.max(result.durationMs, Date.now() - taskStartedAt),
|
|
1085
|
-
tokens: Math.max(cumulativeTokens, accumulatedUsage.totalTokens),
|
|
1086
|
-
usage: accumulatedUsage,
|
|
1087
|
-
},
|
|
1088
1232
|
task,
|
|
1089
|
-
|
|
1233
|
+
p,
|
|
1234
|
+
resolveAttemptTiming(task, timing),
|
|
1090
1235
|
);
|
|
1236
|
+
return finishTask(env, p, settled.result, settled.retries);
|
|
1091
1237
|
} catch (err) {
|
|
1092
|
-
// Any acquired session is released by
|
|
1238
|
+
// Any acquired session is released by runTaskAttempt's finally before an
|
|
1093
1239
|
// exception reaches this boundary. This outer catch handles unexpected
|
|
1094
1240
|
// throws before retry accounting is initialized, so report a minimal
|
|
1095
1241
|
// failure without accumulated usage.
|
|
@@ -1097,7 +1243,6 @@ async function runResolvedTaskCore(
|
|
|
1097
1243
|
env,
|
|
1098
1244
|
p,
|
|
1099
1245
|
failTask(task, err instanceof Error ? err.message : String(err)),
|
|
1100
|
-
task,
|
|
1101
1246
|
);
|
|
1102
1247
|
}
|
|
1103
1248
|
}
|