@the-open-engine/zeroshot 6.10.0 → 6.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +16 -0
- package/cluster-hooks/block-ask-user-question.py +2 -3
- package/cluster-hooks/block-dangerous-git.py +2 -3
- package/lib/agent-cli-provider/adapters/claude.d.ts.map +1 -1
- package/lib/agent-cli-provider/adapters/claude.js +42 -1
- package/lib/agent-cli-provider/adapters/claude.js.map +1 -1
- package/lib/agent-cli-provider/contract-options.d.ts.map +1 -1
- package/lib/agent-cli-provider/contract-options.js +2 -0
- package/lib/agent-cli-provider/contract-options.js.map +1 -1
- package/lib/agent-cli-provider/types.d.ts +6 -2
- package/lib/agent-cli-provider/types.d.ts.map +1 -1
- package/lib/agent-cli-provider/types.js.map +1 -1
- package/package.json +1 -1
- package/src/agent/agent-lifecycle.js +13 -6
- package/src/agent/agent-task-executor.js +526 -313
- package/src/agent-cli-provider/adapters/claude.ts +46 -1
- package/src/agent-cli-provider/contract-options.ts +6 -0
- package/src/agent-cli-provider/types.ts +9 -6
- package/src/claude-task-runner.js +153 -44
- package/src/task-spawn-cleanup-ownership.js +82 -0
- package/src/worktree-claude-config.js +193 -85
- package/task-lib/attachable-watcher.js +93 -267
- package/task-lib/command-spec-cleanup.js +219 -0
- package/task-lib/commands/clean.js +35 -5
- package/task-lib/commands/get-task-id-by-spawn-token.js +10 -0
- package/task-lib/commands/kill.js +117 -4
- package/task-lib/commands/status.js +1 -0
- package/task-lib/runner.js +61 -4
- package/task-lib/store.js +68 -6
- package/task-lib/watcher-output-runtime.js +284 -0
- package/task-lib/watcher.js +124 -257
|
@@ -15,12 +15,17 @@ const path = require('path');
|
|
|
15
15
|
const fs = require('fs');
|
|
16
16
|
const os = require('os');
|
|
17
17
|
const { parseProviderChunk, getProvider } = require('../providers');
|
|
18
|
-
const { getTask } = require('../../task-lib/store.js');
|
|
18
|
+
const { getTask, getTaskBySpawnOwnershipToken } = require('../../task-lib/store.js');
|
|
19
19
|
const { loadSettings } = require('../../lib/settings.js');
|
|
20
20
|
const { resolveClaudeAuth } = require('../../lib/settings/claude-auth.js');
|
|
21
21
|
const { prependWorktreeToolBinToEnv } = require('../worktree-tooling-env.js');
|
|
22
22
|
const {
|
|
23
|
-
|
|
23
|
+
CLAUDE_SETTINGS_ENV,
|
|
24
|
+
cleanupClaudeSettingsOverlay,
|
|
25
|
+
ensureAskUserQuestionHook,
|
|
26
|
+
ensureDangerousGitHook,
|
|
27
|
+
prepareClaudeSettingsOverlay,
|
|
28
|
+
resolveContainerMcpConfigPath,
|
|
24
29
|
resolveRepoMcpConfigPath,
|
|
25
30
|
} = require('../worktree-claude-config.js');
|
|
26
31
|
const {
|
|
@@ -28,6 +33,14 @@ const {
|
|
|
28
33
|
wrapTaskRunWithIsolatedSettings,
|
|
29
34
|
} = require('../task-run-model-args.js');
|
|
30
35
|
const { buildRawLogOnlyMetadata } = require('./context-replay-policy');
|
|
36
|
+
const {
|
|
37
|
+
TASK_SPAWN_OWNERSHIP_TOKEN_ENV,
|
|
38
|
+
cleanupCallerOwnedCommand,
|
|
39
|
+
callerOwnsCommandCleanup,
|
|
40
|
+
createTaskSpawnOwnershipToken,
|
|
41
|
+
requireTaskIdFromWrapperResult,
|
|
42
|
+
trackTaskWrapperCleanupOwnership,
|
|
43
|
+
} = require('../task-spawn-cleanup-ownership');
|
|
31
44
|
const {
|
|
32
45
|
providerSessionFromCompletedTask,
|
|
33
46
|
resolveAgentResumeSessionId,
|
|
@@ -378,10 +391,6 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false,
|
|
|
378
391
|
return sanitizeErrorMessage(`Task failed. Output: ${trimmedOutput}`);
|
|
379
392
|
}
|
|
380
393
|
|
|
381
|
-
// Track which config dirs already have zeroshot-installed hooks.
|
|
382
|
-
const askUserQuestionHookInstalledDirs = new Set();
|
|
383
|
-
const dangerousGitHookInstalledDirs = new Set();
|
|
384
|
-
|
|
385
394
|
/**
|
|
386
395
|
* Extract token usage from NDJSON output.
|
|
387
396
|
* Looks for the 'result' event line which contains usage data.
|
|
@@ -435,162 +444,6 @@ function extractTokenUsage(output, providerName = 'claude') {
|
|
|
435
444
|
};
|
|
436
445
|
}
|
|
437
446
|
|
|
438
|
-
/**
|
|
439
|
-
* Ensure the AskUserQuestion blocking hook is installed in user's Claude config.
|
|
440
|
-
* This adds defense-in-depth by blocking the tool at the Claude CLI level.
|
|
441
|
-
* Modifies ~/.claude/settings.json and copies hook script to ~/.claude/hooks/
|
|
442
|
-
*
|
|
443
|
-
* Safe to call multiple times - only modifies config once per process.
|
|
444
|
-
*/
|
|
445
|
-
function ensureAskUserQuestionHook(targetClaudeDir = null) {
|
|
446
|
-
const userClaudeDir =
|
|
447
|
-
targetClaudeDir || process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
|
|
448
|
-
if (askUserQuestionHookInstalledDirs.has(userClaudeDir)) {
|
|
449
|
-
return;
|
|
450
|
-
}
|
|
451
|
-
const hooksDir = path.join(userClaudeDir, 'hooks');
|
|
452
|
-
const settingsPath = path.join(userClaudeDir, 'settings.json');
|
|
453
|
-
const hookScriptName = 'block-ask-user-question.py';
|
|
454
|
-
const hookScriptDst = path.join(hooksDir, hookScriptName);
|
|
455
|
-
|
|
456
|
-
// Ensure hooks directory exists
|
|
457
|
-
if (!fs.existsSync(hooksDir)) {
|
|
458
|
-
fs.mkdirSync(hooksDir, { recursive: true });
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
// Copy hook script if not present or outdated
|
|
462
|
-
const hookScriptSrc = path.join(__dirname, '..', '..', 'cluster-hooks', hookScriptName);
|
|
463
|
-
if (fs.existsSync(hookScriptSrc)) {
|
|
464
|
-
// Always copy to ensure latest version
|
|
465
|
-
fs.copyFileSync(hookScriptSrc, hookScriptDst);
|
|
466
|
-
fs.chmodSync(hookScriptDst, 0o755);
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
// Read existing settings or create new
|
|
470
|
-
let settings = {};
|
|
471
|
-
if (fs.existsSync(settingsPath)) {
|
|
472
|
-
try {
|
|
473
|
-
settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
|
474
|
-
} catch (e) {
|
|
475
|
-
console.warn(`[AgentTaskExecutor] Could not parse settings.json, creating new: ${e.message}`);
|
|
476
|
-
settings = {};
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
// Ensure hooks structure exists
|
|
481
|
-
if (!settings.hooks) {
|
|
482
|
-
settings.hooks = {};
|
|
483
|
-
}
|
|
484
|
-
if (!settings.hooks.PreToolUse) {
|
|
485
|
-
settings.hooks.PreToolUse = [];
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// Check if AskUserQuestion hook already exists
|
|
489
|
-
const hasHook = settings.hooks.PreToolUse.some(
|
|
490
|
-
(entry) =>
|
|
491
|
-
entry.matcher === 'AskUserQuestion' ||
|
|
492
|
-
(entry.hooks && entry.hooks.some((h) => h.command && h.command.includes(hookScriptName)))
|
|
493
|
-
);
|
|
494
|
-
|
|
495
|
-
if (!hasHook) {
|
|
496
|
-
// Add the hook
|
|
497
|
-
settings.hooks.PreToolUse.push({
|
|
498
|
-
matcher: 'AskUserQuestion',
|
|
499
|
-
hooks: [
|
|
500
|
-
{
|
|
501
|
-
type: 'command',
|
|
502
|
-
command: hookScriptDst,
|
|
503
|
-
},
|
|
504
|
-
],
|
|
505
|
-
});
|
|
506
|
-
|
|
507
|
-
// Write updated settings
|
|
508
|
-
fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2));
|
|
509
|
-
console.log(`[AgentTaskExecutor] Installed AskUserQuestion blocking hook in ${settingsPath}`);
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
askUserQuestionHookInstalledDirs.add(userClaudeDir);
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
/**
|
|
516
|
-
* Ensure the dangerous git blocking hook is installed in user's Claude config.
|
|
517
|
-
* This blocks dangerous git commands like stash, checkout --, reset --hard, etc.
|
|
518
|
-
* Modifies ~/.claude/settings.json and copies hook script to ~/.claude/hooks/
|
|
519
|
-
*
|
|
520
|
-
* Only used in worktree mode - Docker isolation mode has its own git-safe.sh wrapper.
|
|
521
|
-
* Safe to call multiple times - only modifies config once per process.
|
|
522
|
-
*/
|
|
523
|
-
function ensureDangerousGitHook(targetClaudeDir = null) {
|
|
524
|
-
const userClaudeDir =
|
|
525
|
-
targetClaudeDir || process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
|
|
526
|
-
if (dangerousGitHookInstalledDirs.has(userClaudeDir)) {
|
|
527
|
-
return;
|
|
528
|
-
}
|
|
529
|
-
const hooksDir = path.join(userClaudeDir, 'hooks');
|
|
530
|
-
const settingsPath = path.join(userClaudeDir, 'settings.json');
|
|
531
|
-
const hookScriptName = 'block-dangerous-git.py';
|
|
532
|
-
const hookScriptDst = path.join(hooksDir, hookScriptName);
|
|
533
|
-
|
|
534
|
-
// Ensure hooks directory exists
|
|
535
|
-
if (!fs.existsSync(hooksDir)) {
|
|
536
|
-
fs.mkdirSync(hooksDir, { recursive: true });
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
// Copy hook script if not present or outdated
|
|
540
|
-
const hookScriptSrc = path.join(__dirname, '..', '..', 'cluster-hooks', hookScriptName);
|
|
541
|
-
if (fs.existsSync(hookScriptSrc)) {
|
|
542
|
-
// Always copy to ensure latest version
|
|
543
|
-
fs.copyFileSync(hookScriptSrc, hookScriptDst);
|
|
544
|
-
fs.chmodSync(hookScriptDst, 0o755);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
// Read existing settings or create new
|
|
548
|
-
let settings = {};
|
|
549
|
-
if (fs.existsSync(settingsPath)) {
|
|
550
|
-
try {
|
|
551
|
-
settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
|
552
|
-
} catch (e) {
|
|
553
|
-
console.warn(`[AgentTaskExecutor] Could not parse settings.json, creating new: ${e.message}`);
|
|
554
|
-
settings = {};
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
// Ensure hooks structure exists
|
|
559
|
-
if (!settings.hooks) {
|
|
560
|
-
settings.hooks = {};
|
|
561
|
-
}
|
|
562
|
-
if (!settings.hooks.PreToolUse) {
|
|
563
|
-
settings.hooks.PreToolUse = [];
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
// Check if dangerous git hook already exists
|
|
567
|
-
const hasHook = settings.hooks.PreToolUse.some(
|
|
568
|
-
(entry) =>
|
|
569
|
-
entry.matcher === 'Bash' &&
|
|
570
|
-
entry.hooks &&
|
|
571
|
-
entry.hooks.some((h) => h.command && h.command.includes(hookScriptName))
|
|
572
|
-
);
|
|
573
|
-
|
|
574
|
-
if (!hasHook) {
|
|
575
|
-
// Add the hook - matches Bash tool to check for dangerous git commands
|
|
576
|
-
settings.hooks.PreToolUse.push({
|
|
577
|
-
matcher: 'Bash',
|
|
578
|
-
hooks: [
|
|
579
|
-
{
|
|
580
|
-
type: 'command',
|
|
581
|
-
command: hookScriptDst,
|
|
582
|
-
},
|
|
583
|
-
],
|
|
584
|
-
});
|
|
585
|
-
|
|
586
|
-
// Write updated settings
|
|
587
|
-
fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2));
|
|
588
|
-
console.log(`[AgentTaskExecutor] Installed dangerous git blocking hook in ${settingsPath}`);
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
dangerousGitHookInstalledDirs.add(userClaudeDir);
|
|
592
|
-
}
|
|
593
|
-
|
|
594
447
|
/**
|
|
595
448
|
* Spawn claude-zeroshots process and stream output via message bus
|
|
596
449
|
* @param {Object} agent - Agent instance
|
|
@@ -650,37 +503,43 @@ async function spawnClaudeTask(agent, context) {
|
|
|
650
503
|
return spawnClaudeTaskIsolated(agent, context);
|
|
651
504
|
}
|
|
652
505
|
|
|
653
|
-
// NON-ISOLATION MODE:
|
|
506
|
+
// NON-ISOLATION MODE: Load safety hooks through an additional per-run settings file. Claude
|
|
507
|
+
// still reads the user's normal config and the repository's project/local config.
|
|
654
508
|
// AskUserQuestion blocking handled via:
|
|
655
509
|
// 1. Prompt injection (see agent-context-builder)
|
|
656
510
|
// 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
|
|
657
|
-
const
|
|
511
|
+
const claudeSettingsPath =
|
|
658
512
|
providerName === 'claude'
|
|
659
|
-
?
|
|
660
|
-
|
|
661
|
-
worktreePath: agent.worktree?.path || null,
|
|
513
|
+
? prepareClaudeSettingsOverlay({
|
|
514
|
+
includeDangerousGit: Boolean(agent.worktree?.enabled),
|
|
662
515
|
})
|
|
663
516
|
: null;
|
|
664
517
|
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
agent,
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
518
|
+
let taskId;
|
|
519
|
+
let pendingLaunch;
|
|
520
|
+
try {
|
|
521
|
+
const spawnEnv = buildSpawnEnv(agent, providerName, modelSpec, { claudeSettingsPath });
|
|
522
|
+
taskId = await spawnTaskProcess({
|
|
523
|
+
agent,
|
|
524
|
+
ctPath,
|
|
525
|
+
args,
|
|
526
|
+
cwd,
|
|
527
|
+
spawnEnv,
|
|
528
|
+
});
|
|
529
|
+
pendingLaunch = agent.currentTask;
|
|
530
|
+
} catch (error) {
|
|
531
|
+
cleanupCallerOwnedCommand(error, () => cleanupClaudeSettingsOverlay(claudeSettingsPath));
|
|
532
|
+
throw error;
|
|
533
|
+
}
|
|
674
534
|
|
|
535
|
+
// The task ID transfers provider and cleanup ownership to the detached
|
|
536
|
+
// watcher. From this point, follower/PID observation failures must not remove
|
|
537
|
+
// files that a live provider still reads.
|
|
675
538
|
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
|
|
676
|
-
|
|
677
|
-
// Wait for task to be registered in zeroshot storage (race condition fix)
|
|
678
539
|
await waitForTaskReady(agent, taskId);
|
|
540
|
+
if (pendingLaunch?.cancelled) throw new Error(`Task launch cancelled: ${taskId}`);
|
|
679
541
|
|
|
680
|
-
|
|
681
|
-
// The watcher spawns the actual CLI and writes PID to SQLite asynchronously.
|
|
682
|
-
// We must poll because the watcher runs in a forked process.
|
|
683
|
-
const MAX_PID_POLLS = 30; // 3 seconds max
|
|
542
|
+
const MAX_PID_POLLS = 30;
|
|
684
543
|
const PID_POLL_DELAY = 100;
|
|
685
544
|
let realPid = null;
|
|
686
545
|
let terminalBeforePidObservation = false;
|
|
@@ -698,6 +557,8 @@ async function spawnClaudeTask(agent, context) {
|
|
|
698
557
|
await new Promise((r) => setTimeout(r, PID_POLL_DELAY));
|
|
699
558
|
}
|
|
700
559
|
|
|
560
|
+
if (pendingLaunch?.cancelled) throw new Error(`Task launch cancelled: ${taskId}`);
|
|
561
|
+
|
|
701
562
|
if (realPid) {
|
|
702
563
|
agent.processPid = realPid;
|
|
703
564
|
agent._publishLifecycle('PROCESS_SPAWNED', { pid: realPid });
|
|
@@ -708,7 +569,6 @@ async function spawnClaudeTask(agent, context) {
|
|
|
708
569
|
agent._log(`⚠️ Agent ${agent.id}: PID not available (task may use non-standard watcher)`);
|
|
709
570
|
}
|
|
710
571
|
|
|
711
|
-
// Now follow the logs and stream output
|
|
712
572
|
return followClaudeTaskLogs(agent, taskId);
|
|
713
573
|
}
|
|
714
574
|
|
|
@@ -749,9 +609,9 @@ function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) {
|
|
|
749
609
|
args.push('--json-schema', schema);
|
|
750
610
|
}
|
|
751
611
|
|
|
752
|
-
// MCP servers:
|
|
753
|
-
//
|
|
754
|
-
//
|
|
612
|
+
// MCP servers: explicitly forward the repo config through the task CLI.
|
|
613
|
+
// Claude receives the path; providers such as Copilot receive inlined JSON
|
|
614
|
+
// so it survives container path translation.
|
|
755
615
|
for (const mcpArg of resolveMcpConfigArgs(agent, providerName)) {
|
|
756
616
|
args.push(mcpArg);
|
|
757
617
|
}
|
|
@@ -767,20 +627,28 @@ function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) {
|
|
|
767
627
|
/**
|
|
768
628
|
* Build the `--mcp-config` args for a task-run invocation, or [] when they don't apply.
|
|
769
629
|
*
|
|
770
|
-
*
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
* under local, worktree, and Docker isolation without host/container path translation.
|
|
630
|
+
* Claude receives the config path because its CLI supports `--mcp-config`.
|
|
631
|
+
* Other providers whose adapter models an MCP config flag receive inlined
|
|
632
|
+
* content so the identical value works under Docker isolation.
|
|
774
633
|
*/
|
|
775
634
|
function resolveMcpConfigArgs(agent, providerName) {
|
|
776
|
-
if (!providerModelsMcpConfigFlag(providerName)) return [];
|
|
777
|
-
|
|
778
635
|
const mcpPath = resolveRepoMcpConfigPath({
|
|
779
636
|
cwd: agent.config?.cwd || process.cwd(),
|
|
780
637
|
worktreePath: agent.worktree?.path || null,
|
|
781
638
|
});
|
|
782
639
|
if (!mcpPath) return [];
|
|
783
640
|
|
|
641
|
+
if (providerName === 'claude') {
|
|
642
|
+
const forwardedPath = agent.isolation?.enabled
|
|
643
|
+
? resolveContainerMcpConfigPath({
|
|
644
|
+
cwd: agent.config?.cwd || process.cwd(),
|
|
645
|
+
worktreePath: agent.worktree?.path || null,
|
|
646
|
+
})
|
|
647
|
+
: mcpPath;
|
|
648
|
+
return forwardedPath ? ['--mcp-config', forwardedPath] : [];
|
|
649
|
+
}
|
|
650
|
+
if (!providerModelsMcpConfigFlag(providerName)) return [];
|
|
651
|
+
|
|
784
652
|
const content = fs.readFileSync(mcpPath, 'utf8').trim();
|
|
785
653
|
if (content.length === 0) return [];
|
|
786
654
|
|
|
@@ -820,21 +688,8 @@ function buildFinalContext({ agent, context, desiredOutputFormat, runOutputForma
|
|
|
820
688
|
return context;
|
|
821
689
|
}
|
|
822
690
|
|
|
823
|
-
function ensureProviderHooks(agent, providerName, claudeConfigDir = null) {
|
|
824
|
-
if (providerName !== 'claude') {
|
|
825
|
-
return;
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
ensureAskUserQuestionHook(claudeConfigDir);
|
|
829
|
-
|
|
830
|
-
// WORKTREE MODE: Install git safety hook (blocks dangerous git commands)
|
|
831
|
-
if (agent.worktree?.enabled) {
|
|
832
|
-
ensureDangerousGitHook(claudeConfigDir);
|
|
833
|
-
}
|
|
834
|
-
}
|
|
835
|
-
|
|
836
691
|
function buildSpawnEnv(agent, providerName, modelSpec, options = {}) {
|
|
837
|
-
const {
|
|
692
|
+
const { claudeSettingsPath = null } = options;
|
|
838
693
|
const spawnEnv = { ...process.env };
|
|
839
694
|
const agentCwd = agent.config?.cwd || agent.worktree?.path || process.cwd();
|
|
840
695
|
const clusterId = agent.cluster?.id || agent.cluster_id || process.env.ZEROSHOT_CLUSTER_ID;
|
|
@@ -859,8 +714,8 @@ function buildSpawnEnv(agent, providerName, modelSpec, options = {}) {
|
|
|
859
714
|
|
|
860
715
|
if (providerName === 'claude') {
|
|
861
716
|
Object.assign(spawnEnv, buildClaudeEnv(modelSpec));
|
|
862
|
-
if (
|
|
863
|
-
spawnEnv
|
|
717
|
+
if (claudeSettingsPath) {
|
|
718
|
+
spawnEnv[CLAUDE_SETTINGS_ENV] = claudeSettingsPath;
|
|
864
719
|
}
|
|
865
720
|
|
|
866
721
|
// WORKTREE MODE: Activate git safety hook via environment variable
|
|
@@ -882,21 +737,136 @@ function parseTaskIdFromOutput(stdout) {
|
|
|
882
737
|
return match ? match[1] : null;
|
|
883
738
|
}
|
|
884
739
|
|
|
885
|
-
|
|
886
|
-
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
887
|
-
const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
|
|
740
|
+
const TASK_TERMINAL_STATUSES = new Set(['completed', 'failed', 'killed', 'stale']);
|
|
888
741
|
|
|
742
|
+
function assignDurableTaskId(agent, taskId) {
|
|
743
|
+
if (!taskId || agent.currentTaskId === taskId) return;
|
|
744
|
+
agent.currentTaskId = taskId;
|
|
745
|
+
agent._publishLifecycle('TASK_ID_ASSIGNED', {
|
|
746
|
+
pid: agent.processPid,
|
|
747
|
+
taskId,
|
|
748
|
+
});
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
function createPendingTaskLaunchHandle({
|
|
752
|
+
agent,
|
|
753
|
+
proc,
|
|
754
|
+
ctPath,
|
|
755
|
+
findPersistedTaskId,
|
|
756
|
+
waitForWrapperClose,
|
|
757
|
+
isWrapperClosed,
|
|
758
|
+
}) {
|
|
759
|
+
let cancellation = null;
|
|
760
|
+
const handle = {
|
|
761
|
+
pendingLaunch: true,
|
|
762
|
+
cancelled: false,
|
|
763
|
+
kill(reason = 'Task killed') {
|
|
764
|
+
handle.cancelled = true;
|
|
765
|
+
if (cancellation) return cancellation;
|
|
766
|
+
const cancellationAttempt = (async () => {
|
|
767
|
+
let taskId = findPersistedTaskId();
|
|
768
|
+
let commandError = null;
|
|
769
|
+
let commandAttempted = false;
|
|
770
|
+
const cancelTask = async (persistedTaskId) => {
|
|
771
|
+
commandAttempted = true;
|
|
772
|
+
assignDurableTaskId(agent, persistedTaskId);
|
|
773
|
+
try {
|
|
774
|
+
await runCommandWithTimeout(ctPath, ['kill', persistedTaskId], { timeout: 10000 });
|
|
775
|
+
} catch (error) {
|
|
776
|
+
commandError = error;
|
|
777
|
+
}
|
|
778
|
+
};
|
|
779
|
+
if (taskId) await cancelTask(taskId);
|
|
780
|
+
|
|
781
|
+
if (!isWrapperClosed()) {
|
|
782
|
+
proc.kill('SIGKILL');
|
|
783
|
+
await waitForWrapperClose;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
taskId = taskId || findPersistedTaskId();
|
|
787
|
+
if (taskId && !commandAttempted) await cancelTask(taskId);
|
|
788
|
+
if (taskId && commandError === null) {
|
|
789
|
+
const task = getTask(taskId);
|
|
790
|
+
if (!task || !TASK_TERMINAL_STATUSES.has(task.status) || task.commandCleanup) {
|
|
791
|
+
commandError = new Error(
|
|
792
|
+
`Task ${taskId} termination and command cleanup were not confirmed`
|
|
793
|
+
);
|
|
794
|
+
}
|
|
795
|
+
} else if (taskId) {
|
|
796
|
+
const task = getTask(taskId);
|
|
797
|
+
if (task && TASK_TERMINAL_STATUSES.has(task.status) && !task.commandCleanup) {
|
|
798
|
+
commandError = null;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
if (commandError) {
|
|
803
|
+
return { forced: false, reason: commandError.message };
|
|
804
|
+
}
|
|
805
|
+
agent._log?.(`Cancelled pending task launch${taskId ? ` ${taskId}` : ''}: ${reason}`);
|
|
806
|
+
return { forced: true, taskId: taskId || null };
|
|
807
|
+
})();
|
|
808
|
+
cancellation = cancellationAttempt;
|
|
809
|
+
cancellationAttempt.then(
|
|
810
|
+
(termination) => {
|
|
811
|
+
if (termination?.forced === false && cancellation === cancellationAttempt) {
|
|
812
|
+
cancellation = null;
|
|
813
|
+
}
|
|
814
|
+
},
|
|
815
|
+
() => {
|
|
816
|
+
if (cancellation === cancellationAttempt) cancellation = null;
|
|
817
|
+
}
|
|
818
|
+
);
|
|
819
|
+
return cancellationAttempt;
|
|
820
|
+
},
|
|
821
|
+
};
|
|
822
|
+
return handle;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
function spawnTaskProcess({
|
|
826
|
+
agent,
|
|
827
|
+
ctPath,
|
|
828
|
+
args,
|
|
829
|
+
cwd,
|
|
830
|
+
spawnEnv,
|
|
831
|
+
spawnTimeoutMs = 30000,
|
|
832
|
+
}) {
|
|
833
|
+
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it.
|
|
834
|
+
const SPAWN_TIMEOUT_MS = spawnTimeoutMs;
|
|
889
835
|
// spawn() throws on null bytes in argv; strip them before they get there.
|
|
890
836
|
const safeArgs = args.map((arg) => (typeof arg === 'string' ? arg.replace(/\0/g, '') : arg));
|
|
837
|
+
const ownershipToken = createTaskSpawnOwnershipToken();
|
|
838
|
+
const findPersistedTaskId = () => getTaskBySpawnOwnershipToken(ownershipToken)?.id || null;
|
|
891
839
|
|
|
892
840
|
return new Promise((resolve, reject) => {
|
|
893
841
|
const proc = spawn(ctPath, safeArgs, {
|
|
894
842
|
cwd,
|
|
895
843
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
896
|
-
env: spawnEnv,
|
|
844
|
+
env: { ...spawnEnv, [TASK_SPAWN_OWNERSHIP_TOKEN_ENV]: ownershipToken },
|
|
897
845
|
windowsHide: true,
|
|
898
846
|
});
|
|
899
847
|
|
|
848
|
+
let wrapperClosed = false;
|
|
849
|
+
let resolveWrapperClose;
|
|
850
|
+
const waitForWrapperClose = new Promise((resolveClose) => {
|
|
851
|
+
resolveWrapperClose = resolveClose;
|
|
852
|
+
});
|
|
853
|
+
const markWrapperClosed = () => {
|
|
854
|
+
if (wrapperClosed) return;
|
|
855
|
+
wrapperClosed = true;
|
|
856
|
+
resolveWrapperClose();
|
|
857
|
+
};
|
|
858
|
+
proc.once('close', markWrapperClosed);
|
|
859
|
+
proc.once('error', markWrapperClosed);
|
|
860
|
+
const pendingLaunch = createPendingTaskLaunchHandle({
|
|
861
|
+
agent,
|
|
862
|
+
proc,
|
|
863
|
+
ctPath,
|
|
864
|
+
findPersistedTaskId,
|
|
865
|
+
waitForWrapperClose,
|
|
866
|
+
isWrapperClosed: () => wrapperClosed,
|
|
867
|
+
});
|
|
868
|
+
agent.currentTask = pendingLaunch;
|
|
869
|
+
|
|
900
870
|
// NOTE: Don't emit PROCESS_SPAWNED here - proc.pid is a wrapper that exits immediately.
|
|
901
871
|
// Real PID comes from task store after watcher spawns the actual CLI process.
|
|
902
872
|
// PROCESS_SPAWNED is emitted in spawnClaudeTask after waitForTaskReady + PID polling.
|
|
@@ -904,18 +874,49 @@ function spawnTaskProcess({ agent, ctPath, args, cwd, spawnEnv }) {
|
|
|
904
874
|
let stdout = '';
|
|
905
875
|
let stderr = '';
|
|
906
876
|
let resolved = false;
|
|
877
|
+
let timeoutError = null;
|
|
878
|
+
const classifyCleanupOwnership = trackTaskWrapperCleanupOwnership(findPersistedTaskId);
|
|
879
|
+
const rejectWithOwnership = async (error) => {
|
|
880
|
+
const classifiedError = classifyCleanupOwnership(error);
|
|
881
|
+
if (
|
|
882
|
+
!callerOwnsCommandCleanup(classifiedError) &&
|
|
883
|
+
agent.currentTask === pendingLaunch
|
|
884
|
+
) {
|
|
885
|
+
let termination;
|
|
886
|
+
try {
|
|
887
|
+
termination = await pendingLaunch.kill(classifiedError.message);
|
|
888
|
+
} catch (cleanupError) {
|
|
889
|
+
termination = { forced: false, reason: cleanupError.message };
|
|
890
|
+
}
|
|
891
|
+
if (termination?.forced === false) {
|
|
892
|
+
classifiedError.message += ` Task cleanup was not confirmed: ${termination.reason}`;
|
|
893
|
+
classifiedError.retainTaskHandle = true;
|
|
894
|
+
classifiedError.permanent = true;
|
|
895
|
+
classifiedError.restartExhausted = true;
|
|
896
|
+
classifiedError.terminationExhausted = true;
|
|
897
|
+
classifiedError.terminationAttempts = 1;
|
|
898
|
+
classifiedError.taskId = agent.currentTaskId || null;
|
|
899
|
+
} else if (agent.currentTask === pendingLaunch) {
|
|
900
|
+
agent.currentTask = null;
|
|
901
|
+
agent.currentTaskId = null;
|
|
902
|
+
agent.processPid = null;
|
|
903
|
+
agent.lastOutputTime = null;
|
|
904
|
+
agent.taskStartedAt = null;
|
|
905
|
+
}
|
|
906
|
+
} else if (wrapperClosed && agent.currentTask === pendingLaunch) {
|
|
907
|
+
agent.currentTask = null;
|
|
908
|
+
}
|
|
909
|
+
reject(classifiedError);
|
|
910
|
+
};
|
|
907
911
|
|
|
908
912
|
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
|
|
909
913
|
const spawnTimeout = setTimeout(() => {
|
|
910
914
|
if (resolved) return;
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
new Error(
|
|
915
|
-
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
916
|
-
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
917
|
-
)
|
|
915
|
+
timeoutError = new Error(
|
|
916
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
917
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
918
918
|
);
|
|
919
|
+
proc.kill('SIGKILL');
|
|
919
920
|
}, SPAWN_TIMEOUT_MS);
|
|
920
921
|
|
|
921
922
|
proc.stdout.on('data', (data) => {
|
|
@@ -926,48 +927,53 @@ function spawnTaskProcess({ agent, ctPath, args, cwd, spawnEnv }) {
|
|
|
926
927
|
stderr += data.toString();
|
|
927
928
|
});
|
|
928
929
|
|
|
929
|
-
proc.on('close', (code, signal) => {
|
|
930
|
+
proc.on('close', async (code, signal) => {
|
|
930
931
|
clearTimeout(spawnTimeout);
|
|
931
932
|
if (resolved) return;
|
|
932
933
|
resolved = true;
|
|
934
|
+
if (timeoutError) {
|
|
935
|
+
await rejectWithOwnership(timeoutError);
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
933
938
|
// Handle process killed by signal (e.g., SIGTERM, SIGKILL, SIGSTOP)
|
|
934
939
|
if (signal) {
|
|
935
|
-
|
|
940
|
+
await rejectWithOwnership(
|
|
941
|
+
new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`)
|
|
942
|
+
);
|
|
936
943
|
return;
|
|
937
944
|
}
|
|
938
945
|
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
946
|
+
let spawnedTaskId;
|
|
947
|
+
try {
|
|
948
|
+
spawnedTaskId = requireTaskIdFromWrapperResult({
|
|
949
|
+
code,
|
|
950
|
+
stdout,
|
|
951
|
+
stderr,
|
|
952
|
+
parseTaskId: parseTaskIdFromOutput,
|
|
953
|
+
persistedTaskId: findPersistedTaskId(),
|
|
954
|
+
});
|
|
955
|
+
} catch (error) {
|
|
956
|
+
await rejectWithOwnership(error);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
949
959
|
|
|
950
|
-
|
|
951
|
-
if (agent.enableLivenessCheck) {
|
|
952
|
-
agent.taskStartedAt = Date.now();
|
|
953
|
-
agent.lastOutputTime = agent.taskStartedAt;
|
|
954
|
-
agent._startLivenessCheck();
|
|
955
|
-
}
|
|
960
|
+
assignDurableTaskId(agent, spawnedTaskId);
|
|
956
961
|
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
reject(new Error(`zeroshot task run failed with code ${code}: ${stderr}`));
|
|
962
|
+
// Start liveness monitoring
|
|
963
|
+
if (agent.enableLivenessCheck) {
|
|
964
|
+
agent.taskStartedAt = Date.now();
|
|
965
|
+
agent.lastOutputTime = agent.taskStartedAt;
|
|
966
|
+
agent._startLivenessCheck();
|
|
963
967
|
}
|
|
968
|
+
|
|
969
|
+
resolve(spawnedTaskId);
|
|
964
970
|
});
|
|
965
971
|
|
|
966
972
|
proc.on('error', (error) => {
|
|
967
973
|
clearTimeout(spawnTimeout);
|
|
968
974
|
if (resolved) return;
|
|
969
975
|
resolved = true;
|
|
970
|
-
|
|
976
|
+
rejectWithOwnership(error);
|
|
971
977
|
});
|
|
972
978
|
});
|
|
973
979
|
}
|
|
@@ -1371,10 +1377,26 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
|
|
|
1371
1377
|
return true;
|
|
1372
1378
|
}
|
|
1373
1379
|
|
|
1380
|
+
function hasPendingCommandCleanup(statusOutput) {
|
|
1381
|
+
return /Cleanup:\s+pending/i.test(stripAnsiCodes(statusOutput));
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
function retryHostTerminalCleanup({ agent, taskId, state, ctPath }) {
|
|
1385
|
+
if (state.commandCleanupRecoveryPending) return;
|
|
1386
|
+
state.commandCleanupRecoveryPending = true;
|
|
1387
|
+
runCommandWithTimeout(ctPath, ['kill', taskId], { timeout: 10000 }, (error) => {
|
|
1388
|
+
state.commandCleanupRecoveryPending = false;
|
|
1389
|
+
if (error) {
|
|
1390
|
+
agent._log(`[${agent.id}] Terminal command cleanup recovery will retry: ${error.message}`);
|
|
1391
|
+
}
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1374
1395
|
function handleStatusCompletion({
|
|
1375
1396
|
agent,
|
|
1376
1397
|
taskId,
|
|
1377
1398
|
providerName,
|
|
1399
|
+
ctPath,
|
|
1378
1400
|
state,
|
|
1379
1401
|
stdout,
|
|
1380
1402
|
pollLogFile,
|
|
@@ -1387,6 +1409,11 @@ function handleStatusCompletion({
|
|
|
1387
1409
|
return false;
|
|
1388
1410
|
}
|
|
1389
1411
|
|
|
1412
|
+
if (hasPendingCommandCleanup(cleanStdout)) {
|
|
1413
|
+
retryHostTerminalCleanup({ agent, taskId, state, ctPath });
|
|
1414
|
+
return true;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1390
1417
|
pollLogFile();
|
|
1391
1418
|
|
|
1392
1419
|
let success = isCompleted;
|
|
@@ -1481,6 +1508,7 @@ function createLogFollower({ agent, taskId, fsModule, ctPath, providerName }) {
|
|
|
1481
1508
|
state.consecutiveExecFailures = 0;
|
|
1482
1509
|
handleStatusCompletion({
|
|
1483
1510
|
agent,
|
|
1511
|
+
ctPath,
|
|
1484
1512
|
taskId,
|
|
1485
1513
|
providerName,
|
|
1486
1514
|
state,
|
|
@@ -1505,7 +1533,7 @@ function createLogFollower({ agent, taskId, fsModule, ctPath, providerName }) {
|
|
|
1505
1533
|
*/
|
|
1506
1534
|
function followClaudeTaskLogs(agent, taskId) {
|
|
1507
1535
|
const fsModule = require('fs');
|
|
1508
|
-
const ctPath = getClaudeTasksPath();
|
|
1536
|
+
const ctPath = agent.taskCliPath || getClaudeTasksPath();
|
|
1509
1537
|
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1510
1538
|
|
|
1511
1539
|
return createLogFollower({ agent, taskId, fsModule, ctPath, providerName });
|
|
@@ -1586,35 +1614,145 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
1586
1614
|
|
|
1587
1615
|
// STEP 1: Spawn task and extract task ID (same as non-isolated mode)
|
|
1588
1616
|
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
1589
|
-
const SPAWN_TIMEOUT_MS = 30000;
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1617
|
+
const SPAWN_TIMEOUT_MS = agent.spawnTimeoutMs ?? 30000;
|
|
1618
|
+
const ownershipToken = createTaskSpawnOwnershipToken();
|
|
1619
|
+
// Auth env vars are injected by IsolationManager; the launch token is the
|
|
1620
|
+
// only authoritative bridge back to the detached task row in the container.
|
|
1621
|
+
const isolatedEnv = {
|
|
1622
|
+
...(providerName === 'claude' ? buildClaudeEnv(modelSpec, { includeAuth: false }) : {}),
|
|
1623
|
+
[TASK_SPAWN_OWNERSHIP_TOKEN_ENV]: ownershipToken,
|
|
1624
|
+
};
|
|
1593
1625
|
|
|
1626
|
+
let isolatedPendingLaunch = null;
|
|
1594
1627
|
const taskId = await new Promise((resolve, reject) => {
|
|
1595
1628
|
const proc = manager.spawnInContainer(clusterId, command, {
|
|
1596
1629
|
env: isolatedEnv,
|
|
1597
1630
|
});
|
|
1598
1631
|
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1632
|
+
let isolatedTaskId = null;
|
|
1633
|
+
let wrapperClosed = false;
|
|
1634
|
+
let resolveWrapperClose;
|
|
1603
1635
|
let stdout = '';
|
|
1604
1636
|
let stderr = '';
|
|
1605
1637
|
let resolved = false;
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1638
|
+
let timeoutError = null;
|
|
1639
|
+
let spawnTimeout = null;
|
|
1640
|
+
let cancellation = null;
|
|
1641
|
+
const waitForWrapperClose = new Promise((resolveClose) => {
|
|
1642
|
+
resolveWrapperClose = resolveClose;
|
|
1643
|
+
});
|
|
1644
|
+
const markWrapperClosed = () => {
|
|
1645
|
+
if (!wrapperClosed) {
|
|
1646
|
+
wrapperClosed = true;
|
|
1647
|
+
resolveWrapperClose();
|
|
1648
|
+
}
|
|
1649
|
+
};
|
|
1650
|
+
const findPersistedTaskId = async () => {
|
|
1651
|
+
const persistedTaskId = await resolveIsolatedTaskIdBySpawnToken(
|
|
1652
|
+
manager,
|
|
1653
|
+
clusterId,
|
|
1654
|
+
ownershipToken
|
|
1655
|
+
);
|
|
1656
|
+
if (persistedTaskId) {
|
|
1657
|
+
isolatedTaskId = persistedTaskId;
|
|
1658
|
+
assignDurableTaskId(agent, persistedTaskId);
|
|
1659
|
+
}
|
|
1660
|
+
return persistedTaskId;
|
|
1661
|
+
};
|
|
1662
|
+
const rejectLaunch = (error, { retainHandle = false } = {}) => {
|
|
1609
1663
|
if (resolved) return;
|
|
1610
1664
|
resolved = true;
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1665
|
+
clearTimeout(spawnTimeout);
|
|
1666
|
+
const rejection = error instanceof Error ? error : new Error(String(error));
|
|
1667
|
+
if (retainHandle) {
|
|
1668
|
+
rejection.retainTaskHandle = true;
|
|
1669
|
+
rejection.permanent = true;
|
|
1670
|
+
rejection.restartExhausted = true;
|
|
1671
|
+
rejection.terminationExhausted = true;
|
|
1672
|
+
rejection.terminationAttempts = 1;
|
|
1673
|
+
rejection.taskId = isolatedTaskId || agent.currentTaskId || null;
|
|
1674
|
+
} else if (agent.currentTask === isolatedPendingLaunch) {
|
|
1675
|
+
agent.currentTask = null;
|
|
1676
|
+
}
|
|
1677
|
+
reject(rejection);
|
|
1678
|
+
};
|
|
1679
|
+
|
|
1680
|
+
proc.once('close', markWrapperClosed);
|
|
1681
|
+
proc.once('error', markWrapperClosed);
|
|
1682
|
+
isolatedPendingLaunch = {
|
|
1683
|
+
pendingLaunch: true,
|
|
1684
|
+
cancelled: false,
|
|
1685
|
+
async kill(reason = 'Task killed') {
|
|
1686
|
+
isolatedPendingLaunch.cancelled = true;
|
|
1687
|
+
if (cancellation) return cancellation;
|
|
1688
|
+
cancellation = (async () => {
|
|
1689
|
+
let termination = null;
|
|
1690
|
+
let commandError = null;
|
|
1691
|
+
try {
|
|
1692
|
+
const persistedTaskId = isolatedTaskId || (await findPersistedTaskId());
|
|
1693
|
+
if (persistedTaskId) {
|
|
1694
|
+
termination = await terminateIsolatedTask(manager, clusterId, persistedTaskId);
|
|
1695
|
+
}
|
|
1696
|
+
} catch (error) {
|
|
1697
|
+
commandError = error;
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
if (!wrapperClosed) {
|
|
1701
|
+
proc.kill('SIGKILL');
|
|
1702
|
+
await waitForWrapperClose;
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
try {
|
|
1706
|
+
const persistedTaskId = isolatedTaskId || (await findPersistedTaskId());
|
|
1707
|
+
if (persistedTaskId && !termination) {
|
|
1708
|
+
termination = await terminateIsolatedTask(manager, clusterId, persistedTaskId);
|
|
1709
|
+
commandError = null;
|
|
1710
|
+
}
|
|
1711
|
+
} catch (error) {
|
|
1712
|
+
commandError ||= error;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
if (commandError) {
|
|
1716
|
+
return { forced: false, reason: commandError.message };
|
|
1717
|
+
}
|
|
1718
|
+
agent._log?.(`Cancelled pending isolated task launch: ${reason}`);
|
|
1719
|
+
return termination
|
|
1720
|
+
? { ...termination, forced: true, taskId: isolatedTaskId }
|
|
1721
|
+
: { forced: true, taskId: null };
|
|
1722
|
+
})();
|
|
1723
|
+
|
|
1724
|
+
const termination = await cancellation;
|
|
1725
|
+
if (termination?.forced === false) cancellation = null;
|
|
1726
|
+
if (!resolved) {
|
|
1727
|
+
const error =
|
|
1728
|
+
timeoutError ||
|
|
1729
|
+
new Error(
|
|
1730
|
+
termination?.forced === false
|
|
1731
|
+
? `Task launch cancellation was not confirmed: ${termination.reason}`
|
|
1732
|
+
: `Task launch cancelled: ${isolatedTaskId || 'before persistence'}`
|
|
1733
|
+
);
|
|
1734
|
+
rejectLaunch(error, { retainHandle: termination?.forced === false });
|
|
1735
|
+
}
|
|
1736
|
+
return termination;
|
|
1737
|
+
},
|
|
1738
|
+
};
|
|
1739
|
+
agent.currentTask = isolatedPendingLaunch;
|
|
1740
|
+
|
|
1741
|
+
// Track PID for resource monitoring
|
|
1742
|
+
agent.processPid = proc.pid;
|
|
1743
|
+
agent._publishLifecycle('PROCESS_SPAWNED', { pid: proc.pid });
|
|
1744
|
+
|
|
1745
|
+
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs. Timeout
|
|
1746
|
+
// uses the same cancellation path so a durable child cannot outlive it.
|
|
1747
|
+
spawnTimeout = setTimeout(() => {
|
|
1748
|
+
if (resolved) return;
|
|
1749
|
+
timeoutError = new Error(
|
|
1750
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
1751
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
1617
1752
|
);
|
|
1753
|
+
isolatedPendingLaunch.kill(timeoutError.message).catch((error) => {
|
|
1754
|
+
rejectLaunch(error, { retainHandle: true });
|
|
1755
|
+
});
|
|
1618
1756
|
}, SPAWN_TIMEOUT_MS);
|
|
1619
1757
|
|
|
1620
1758
|
proc.stdout.on('data', (data) => {
|
|
@@ -1625,42 +1763,47 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
1625
1763
|
stderr += data.toString();
|
|
1626
1764
|
});
|
|
1627
1765
|
|
|
1628
|
-
proc.on('close', (code, signal) => {
|
|
1766
|
+
proc.on('close', async (code, signal) => {
|
|
1629
1767
|
clearTimeout(spawnTimeout);
|
|
1630
|
-
if (resolved) return;
|
|
1631
|
-
resolved = true;
|
|
1632
|
-
// Handle process killed by signal
|
|
1768
|
+
if (resolved || isolatedPendingLaunch.cancelled) return;
|
|
1633
1769
|
if (signal) {
|
|
1634
|
-
|
|
1770
|
+
await isolatedPendingLaunch.kill(
|
|
1771
|
+
`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`
|
|
1772
|
+
);
|
|
1635
1773
|
return;
|
|
1636
1774
|
}
|
|
1637
1775
|
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
const spawnedTaskId =
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1776
|
+
try {
|
|
1777
|
+
const persistedTaskId = await findPersistedTaskId();
|
|
1778
|
+
const spawnedTaskId = requireTaskIdFromWrapperResult({
|
|
1779
|
+
code,
|
|
1780
|
+
stdout,
|
|
1781
|
+
stderr,
|
|
1782
|
+
parseTaskId: parseTaskIdFromOutput,
|
|
1783
|
+
persistedTaskId,
|
|
1784
|
+
});
|
|
1785
|
+
isolatedTaskId = spawnedTaskId;
|
|
1786
|
+
assignDurableTaskId(agent, spawnedTaskId);
|
|
1787
|
+
resolved = true;
|
|
1788
|
+
resolve(spawnedTaskId);
|
|
1789
|
+
} catch (error) {
|
|
1790
|
+
const termination = await isolatedPendingLaunch.kill(error.message);
|
|
1791
|
+
if (!resolved) {
|
|
1792
|
+
rejectLaunch(error, { retainHandle: termination?.forced === false });
|
|
1651
1793
|
}
|
|
1652
|
-
} else {
|
|
1653
|
-
reject(new Error(`zeroshot task run failed with code ${code}: ${stderr}`));
|
|
1654
1794
|
}
|
|
1655
1795
|
});
|
|
1656
1796
|
|
|
1657
|
-
proc.on('error', (error) => {
|
|
1797
|
+
proc.on('error', async (error) => {
|
|
1658
1798
|
clearTimeout(spawnTimeout);
|
|
1659
|
-
if (resolved) return;
|
|
1660
|
-
|
|
1661
|
-
|
|
1799
|
+
if (resolved || isolatedPendingLaunch.cancelled) return;
|
|
1800
|
+
const termination = await isolatedPendingLaunch.kill(error.message);
|
|
1801
|
+
if (termination?.forced === false && !resolved) {
|
|
1802
|
+
rejectLaunch(error, { retainHandle: true });
|
|
1803
|
+
}
|
|
1662
1804
|
});
|
|
1663
1805
|
});
|
|
1806
|
+
if (isolatedPendingLaunch?.cancelled) throw new Error(`Task launch cancelled: ${taskId}`);
|
|
1664
1807
|
|
|
1665
1808
|
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId} in container...`);
|
|
1666
1809
|
|
|
@@ -1759,6 +1902,32 @@ function rejectIsolatedFollower({ agent, state, cleanup, reject, error }) {
|
|
|
1759
1902
|
reject(error);
|
|
1760
1903
|
}
|
|
1761
1904
|
|
|
1905
|
+
function rejectIsolatedFollowerRetainingHandle({ state, cleanup, reject, error }) {
|
|
1906
|
+
if (state.resolved || state.failureSettled) return;
|
|
1907
|
+
state.failureSettled = true;
|
|
1908
|
+
cleanup();
|
|
1909
|
+
reject(error);
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1912
|
+
async function resolveIsolatedTaskIdBySpawnToken(manager, clusterId, ownershipToken) {
|
|
1913
|
+
const result = await manager.execInContainer(clusterId, [
|
|
1914
|
+
'zeroshot',
|
|
1915
|
+
'get-task-id-by-spawn-token',
|
|
1916
|
+
ownershipToken,
|
|
1917
|
+
]);
|
|
1918
|
+
if (result.code === 2) return null;
|
|
1919
|
+
if (result.code !== 0) {
|
|
1920
|
+
throw new Error(
|
|
1921
|
+
`Failed to resolve isolated task ownership: ${result.stderr || result.stdout || `exit ${result.code}`}`
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
const taskId = result.stdout.trim();
|
|
1925
|
+
if (!taskId) {
|
|
1926
|
+
throw new Error('Isolated task ownership lookup returned an empty task ID');
|
|
1927
|
+
}
|
|
1928
|
+
return taskId;
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1762
1931
|
function parseIsolatedStatus(output) {
|
|
1763
1932
|
return output.match(/Status:\s+(completed|failed|killed|stale|cancelled)/i)?.[1].toLowerCase();
|
|
1764
1933
|
}
|
|
@@ -1766,23 +1935,27 @@ function parseIsolatedStatus(output) {
|
|
|
1766
1935
|
async function terminateIsolatedTask(manager, clusterId, taskId) {
|
|
1767
1936
|
const before = await manager.execInContainer(clusterId, ['zeroshot', 'status', taskId]);
|
|
1768
1937
|
const beforeStatus = before.code === 0 ? parseIsolatedStatus(before.stdout) : null;
|
|
1769
|
-
|
|
1770
|
-
|
|
1938
|
+
const result = await manager.execInContainer(clusterId, ['zeroshot', 'kill', taskId]);
|
|
1939
|
+
if (result.code !== 0) {
|
|
1940
|
+
throw new Error(
|
|
1941
|
+
`Failed to terminate isolated task ${taskId}: ${result.stderr || result.stdout || `exit ${result.code}`}`
|
|
1942
|
+
);
|
|
1771
1943
|
}
|
|
1772
1944
|
|
|
1773
|
-
const result = await manager.execInContainer(clusterId, ['zeroshot', 'kill', taskId]);
|
|
1774
1945
|
const status = await manager.execInContainer(clusterId, ['zeroshot', 'status', taskId]);
|
|
1775
1946
|
const afterStatus = status.code === 0 ? parseIsolatedStatus(status.stdout) : null;
|
|
1776
1947
|
if (!afterStatus) {
|
|
1777
1948
|
throw new Error(
|
|
1778
|
-
`Failed to
|
|
1949
|
+
`Failed to confirm isolated task ${taskId} after cleanup recovery: ${
|
|
1950
|
+
status.stderr || status.stdout || `exit ${status.code}`
|
|
1951
|
+
}`
|
|
1779
1952
|
);
|
|
1780
1953
|
}
|
|
1781
1954
|
|
|
1782
1955
|
return {
|
|
1783
|
-
alreadyTerminal:
|
|
1784
|
-
forced: afterStatus === 'killed',
|
|
1785
|
-
status: afterStatus,
|
|
1956
|
+
alreadyTerminal: Boolean(beforeStatus),
|
|
1957
|
+
forced: !beforeStatus && afterStatus === 'killed',
|
|
1958
|
+
status: beforeStatus || afterStatus,
|
|
1786
1959
|
};
|
|
1787
1960
|
}
|
|
1788
1961
|
|
|
@@ -1952,7 +2125,7 @@ function buildIsolatedLifecycleHandle({
|
|
|
1952
2125
|
terminate,
|
|
1953
2126
|
kill: terminate,
|
|
1954
2127
|
failClosed(error) {
|
|
1955
|
-
|
|
2128
|
+
rejectIsolatedFollowerRetainingHandle({ state, cleanup, reject, error });
|
|
1956
2129
|
},
|
|
1957
2130
|
};
|
|
1958
2131
|
}
|
|
@@ -2025,6 +2198,7 @@ function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLi
|
|
|
2025
2198
|
});
|
|
2026
2199
|
}
|
|
2027
2200
|
|
|
2201
|
+
|
|
2028
2202
|
async function checkIsolatedStatus({
|
|
2029
2203
|
agent,
|
|
2030
2204
|
manager,
|
|
@@ -2054,6 +2228,28 @@ async function checkIsolatedStatus({
|
|
|
2054
2228
|
return;
|
|
2055
2229
|
}
|
|
2056
2230
|
|
|
2231
|
+
if (status && hasPendingCommandCleanup(statusOutput)) {
|
|
2232
|
+
if (!state.commandCleanupRecoveryPromise) {
|
|
2233
|
+
const recovery = manager.execInContainer(clusterId, ['zeroshot', 'kill', taskId]);
|
|
2234
|
+
state.commandCleanupRecoveryPromise = recovery;
|
|
2235
|
+
try {
|
|
2236
|
+
const result = await recovery;
|
|
2237
|
+
if (result.code !== 0) {
|
|
2238
|
+
agent._log(
|
|
2239
|
+
`[${agent.id}] Isolated terminal command cleanup recovery will retry: ${
|
|
2240
|
+
result.stderr || result.stdout || `exit ${result.code}`
|
|
2241
|
+
}`
|
|
2242
|
+
);
|
|
2243
|
+
}
|
|
2244
|
+
} finally {
|
|
2245
|
+
if (state.commandCleanupRecoveryPromise === recovery) {
|
|
2246
|
+
state.commandCleanupRecoveryPromise = null;
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
return;
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2057
2253
|
state.logFilePath = logFilePath;
|
|
2058
2254
|
await settleIsolatedTerminalStatus({
|
|
2059
2255
|
agent,
|
|
@@ -2353,29 +2549,40 @@ async function killTask(agent, termination = 'Task killed') {
|
|
|
2353
2549
|
const currentTask = agent.currentTask;
|
|
2354
2550
|
const taskId = agent.currentTaskId;
|
|
2355
2551
|
|
|
2552
|
+
if (currentTask?.pendingLaunch && typeof currentTask.kill === 'function') {
|
|
2553
|
+
const pendingTermination = await currentTask.kill(reason, { code });
|
|
2554
|
+
if (pendingTermination?.forced === false) return pendingTermination;
|
|
2555
|
+
agent._stopLivenessCheck?.();
|
|
2556
|
+
agent.currentTask = null;
|
|
2557
|
+
agent.currentTaskId = null;
|
|
2558
|
+
agent.processPid = null;
|
|
2559
|
+
agent.lastOutputTime = null;
|
|
2560
|
+
agent.taskStartedAt = null;
|
|
2561
|
+
return pendingTermination;
|
|
2562
|
+
}
|
|
2563
|
+
|
|
2356
2564
|
if (agent.isolation?.enabled && taskId) {
|
|
2357
2565
|
return killIsolatedTask(agent, currentTask, taskId, reason, code);
|
|
2358
2566
|
}
|
|
2359
2567
|
|
|
2360
|
-
agent._stopLivenessCheck?.();
|
|
2361
|
-
|
|
2362
2568
|
// Kill the underlying task before resolving the local follower. This keeps
|
|
2363
2569
|
// retries from racing a provider process that is still shutting down.
|
|
2364
2570
|
if (taskId) {
|
|
2365
|
-
const ctPath = getClaudeTasksPath();
|
|
2571
|
+
const ctPath = agent.taskCommandPath || getClaudeTasksPath();
|
|
2366
2572
|
try {
|
|
2367
2573
|
// `kill` is a top-level smart command. `task kill` has never existed.
|
|
2368
2574
|
await runCommandWithTimeout(ctPath, ['kill', taskId], { timeout: 10000 });
|
|
2369
2575
|
agent._log?.(`Killed task ${taskId}`);
|
|
2370
2576
|
} catch (error) {
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
agent._log?.(`Note: Could not kill task ${taskId}: ${error.message}`);
|
|
2577
|
+
agent._log?.(`Note: Could not confirm termination for task ${taskId}: ${error.message}`);
|
|
2578
|
+
return { forced: false, reason: error.message };
|
|
2374
2579
|
}
|
|
2375
2580
|
}
|
|
2376
2581
|
|
|
2582
|
+
agent._stopLivenessCheck?.();
|
|
2583
|
+
|
|
2377
2584
|
if (currentTask && typeof currentTask.kill === 'function') {
|
|
2378
|
-
currentTask.kill(reason, { code });
|
|
2585
|
+
await currentTask.kill(reason, { code });
|
|
2379
2586
|
}
|
|
2380
2587
|
|
|
2381
2588
|
agent.currentTask = null;
|
|
@@ -2387,24 +2594,28 @@ async function killTask(agent, termination = 'Task killed') {
|
|
|
2387
2594
|
|
|
2388
2595
|
async function killIsolatedTask(agent, currentTask, taskId, reason, code) {
|
|
2389
2596
|
let termination;
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
currentTask.kill
|
|
2597
|
+
try {
|
|
2598
|
+
if (currentTask && typeof currentTask.terminate === 'function') {
|
|
2599
|
+
termination = await currentTask.terminate(reason, { code });
|
|
2600
|
+
} else {
|
|
2601
|
+
termination = await terminateIsolatedTask(
|
|
2602
|
+
agent.isolation.manager,
|
|
2603
|
+
agent.isolation.clusterId,
|
|
2604
|
+
taskId
|
|
2605
|
+
);
|
|
2606
|
+
if (currentTask && typeof currentTask.kill === 'function') {
|
|
2607
|
+
currentTask.kill(reason, { code });
|
|
2608
|
+
}
|
|
2400
2609
|
}
|
|
2610
|
+
} catch (error) {
|
|
2611
|
+
return { forced: false, reason: error.message };
|
|
2401
2612
|
}
|
|
2402
2613
|
|
|
2403
|
-
|
|
2404
|
-
if (termination?.forced === false) {
|
|
2614
|
+
if (termination?.forced === false && !termination.alreadyTerminal) {
|
|
2405
2615
|
return termination;
|
|
2406
2616
|
}
|
|
2407
2617
|
|
|
2618
|
+
agent._stopLivenessCheck?.();
|
|
2408
2619
|
agent.currentTask = null;
|
|
2409
2620
|
agent.currentTaskId = null;
|
|
2410
2621
|
agent.processPid = null;
|
|
@@ -2415,6 +2626,8 @@ async function killIsolatedTask(agent, currentTask, taskId, reason, code) {
|
|
|
2415
2626
|
|
|
2416
2627
|
module.exports = {
|
|
2417
2628
|
ensureAskUserQuestionHook,
|
|
2629
|
+
ensureDangerousGitHook,
|
|
2630
|
+
resolveMcpConfigArgs,
|
|
2418
2631
|
spawnClaudeTask,
|
|
2419
2632
|
spawnTaskProcess,
|
|
2420
2633
|
followClaudeTaskLogs,
|