agent-relay 3.2.18 → 3.2.22

package/packages/sdk/dist/workflows/runner.js

@@ -5,8 +5,9 @@
  */
 import { spawn as cpSpawn } from 'node:child_process';
 import { randomBytes } from 'node:crypto';
-import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync, } from 'node:fs';
-import { readFile, writeFile } from 'node:fs/promises';
+import { createWriteStream, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync, } from 'node:fs';
+import { readFile, writeFile, unlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
 import path from 'node:path';
 import chalk from 'chalk';
 import { parse as parseYaml } from 'yaml';
@@ -114,6 +115,7 @@ export class WorkflowRunner {
     activeReviewers = new Map();
     /** Structured CLI session reports captured during the current run, keyed by step name. */
     agentReports = new Map();
+    static PTY_TASK_ARG_SIZE_LIMIT = 2 * 1024 * 1024; // 2 MB
     constructor(options = {}) {
         this.db = options.db ?? new InMemoryWorkflowDb();
         this.workspaceId = options.workspaceId ?? 'local';
@@ -1461,34 +1463,46 @@ export class WorkflowRunner {
         });
     }
     /** Resume a previously paused or partially completed run. */
-    async resume(runId, vars) {
+    async resume(runId, vars, config) {
         // Set up abort controller early so callers can abort() even during setup
         this.abortController = new AbortController();
         this.paused = false;
-        const run = await this.db.getRun(runId);
+        let run = await this.db.getRun(runId);
+        let stepStates = new Map();
         if (!run) {
-            throw new Error(`Run "${runId}" not found`);
+            const reconstructed = this.reconstructRunFromCache(runId, config);
+            if (!reconstructed) {
+                throw new Error(`Run "${runId}" not found (no database entry or cached step outputs)`);
+            }
+            this.log('[resume] Reconstructing run from cached step outputs (workflow-runs.jsonl missing)');
+            run = reconstructed.run;
+            stepStates = reconstructed.stepStates;
+            await this.db.insertRun(run);
+            for (const [, state] of stepStates) {
+                await this.db.insertStep(state.row);
+            }
         }
         this.persistRunIdHint(runId);
         if (run.status !== 'running' && run.status !== 'failed') {
             throw new Error(`Run "${runId}" is in status "${run.status}" and cannot be resumed`);
         }
-        const config = vars ? this.resolveVariables(run.config, vars) : run.config;
+        const resolvedConfig = vars ? this.resolveVariables(run.config, vars) : run.config;
         // Resolve path definitions (same as execute()) so workdir lookups work on resume
-        const pathResult = this.resolvePathDefinitions(config.paths, this.cwd);
+        const pathResult = this.resolvePathDefinitions(resolvedConfig.paths, this.cwd);
         if (pathResult.errors.length > 0) {
             throw new Error(`Path validation failed:\n  ${pathResult.errors.join('\n  ')}`);
         }
         this.resolvedPaths = pathResult.resolved;
-        const workflows = config.workflows ?? [];
+        const workflows = resolvedConfig.workflows ?? [];
         const workflow = workflows.find((w) => w.name === run.workflowName);
         if (!workflow) {
             throw new Error(`Workflow "${run.workflowName}" not found in stored config`);
         }
-        const existingSteps = await this.db.getStepsByRunId(runId);
-        const stepStates = new Map();
-        for (const stepRow of existingSteps) {
-            stepStates.set(stepRow.stepName, { row: stepRow });
+        if (stepStates.size === 0) {
+            const existingSteps = await this.db.getStepsByRunId(runId);
+            for (const stepRow of existingSteps) {
+                stepStates.set(stepRow.stepName, { row: stepRow });
+            }
         }
         // Reset failed steps to pending for retry
         for (const [, state] of stepStates) {
@@ -1507,7 +1521,7 @@ export class WorkflowRunner {
         return this.runWorkflowCore({
             run,
             workflow,
-            config,
+            config: resolvedConfig,
             stepStates,
             isResume: true,
         });
@@ -2783,6 +2797,7 @@ export class WorkflowRunner {
                 let ownerOutput;
                 let ownerElapsed;
                 let completionReason;
+                let promptTaskText;
                 if (usesDedicatedOwner) {
                     const result = await this.executeSupervisedAgentStep(step, { specialist: effectiveSpecialist, owner: effectiveOwner, reviewer: reviewDef }, resolvedTask, timeoutMs);
                     specialistOutput = result.specialistOutput;
@@ -2823,13 +2838,19 @@ export class WorkflowRunner {
                                 : undefined,
                         });
                     const output = typeof spawnResult === 'string' ? spawnResult : spawnResult.output;
+                    promptTaskText =
+                        typeof spawnResult === 'string'
+                            ? effectiveOwner.interactive === false
+                                ? undefined
+                                : ownerTask
+                            : spawnResult.promptTaskText ?? ownerTask;
                     lastExitCode = typeof spawnResult === 'string' ? undefined : spawnResult.exitCode;
                     lastExitSignal = typeof spawnResult === 'string' ? undefined : spawnResult.exitSignal;
                     ownerElapsed = Date.now() - ownerStartTime;
                     this.log(`[${step.name}] Owner "${effectiveOwner.name}" exited`);
                     if (usesOwnerFlow) {
                         try {
-                            const completionDecision = this.resolveOwnerCompletionDecision(step, output, output, ownerTask, resolvedTask);
+                            const completionDecision = this.resolveOwnerCompletionDecision(step, output, output, promptTaskText ?? ownerTask, promptTaskText ?? ownerTask);
                             completionReason = completionDecision.completionReason;
                         }
                         catch (error) {
@@ -2864,7 +2885,7 @@ export class WorkflowRunner {
                 // Self-owned interactive steps still need verification fallback so
                 // explicit OWNER_DECISION output is not mandatory for the happy path.
                 if (step.verification && (!usesOwnerFlow || !usesDedicatedOwner) && !completionReason) {
-                    const verificationResult = this.runVerification(step.verification, specialistOutput, step.name, effectiveOwner.interactive === false ? undefined : resolvedTask);
+                    const verificationResult = this.runVerification(step.verification, specialistOutput, step.name, promptTaskText);
                     completionReason = verificationResult.completionReason;
                 }
                 // Retry-style owner decisions are control-flow signals, not terminal success states.
@@ -3120,7 +3141,8 @@ export class WorkflowRunner {
                 detail: `Worker ${workerRuntimeName} exited`,
                 raw: { worker: workerRuntimeName, exitCode: result.exitCode, exitSignal: result.exitSignal },
             });
-            if (step.verification?.type === 'output_contains' && result.output.includes(step.verification.value)) {
+            if (step.verification?.type === 'output_contains' &&
+                this.outputContainsVerificationToken(result.output, step.verification.value, result.promptTaskText)) {
                 this.log(`[${step.name}] Verification gate observed: output contains ${JSON.stringify(step.verification.value)}`);
             }
         })
@@ -3163,8 +3185,9 @@ export class WorkflowRunner {
             const ownerElapsed = Date.now() - ownerStartTime;
             const ownerOutput = ownerResultObj.output;
             this.log(`[${step.name}] Owner "${supervised.owner.name}" exited`);
-            const specialistOutput = (await workerPromise).output;
-            const completionDecision = this.resolveOwnerCompletionDecision(step, ownerOutput, specialistOutput, supervisorTask, resolvedTask);
+            const workerResultObj = await workerPromise;
+            const specialistOutput = workerResultObj.output;
+            const completionDecision = this.resolveOwnerCompletionDecision(step, ownerOutput, specialistOutput, ownerResultObj.promptTaskText ?? supervisorTask, workerResultObj.promptTaskText ?? specialistTask);
             return {
                 specialistOutput,
                 ownerOutput,
@@ -3379,6 +3402,10 @@ export class WorkflowRunner {
     }
     hasOwnerCompletionMarker(step, output, injectedTaskText) {
         const marker = `STEP_COMPLETE:${step.name}`;
+        const strippedOutput = this.stripInjectedTaskEcho(output, injectedTaskText);
+        if (strippedOutput.includes(marker)) {
+            return true;
+        }
         const taskHasMarker = injectedTaskText.includes(marker);
         const first = output.indexOf(marker);
         if (first === -1) {
@@ -3451,6 +3478,49 @@ export class WorkflowRunner {
             .filter((line) => patterns.every((pattern) => !pattern.test(line)))
             .join('\n');
     }
+    stripInjectedTaskEcho(output, injectedTaskText) {
+        if (!injectedTaskText) {
+            return output;
+        }
+        const candidates = [
+            injectedTaskText,
+            injectedTaskText.replace(/\r\n/g, '\n'),
+            injectedTaskText.replace(/\n/g, '\r\n'),
+        ].filter((candidate, index, all) => candidate.length > 0 && all.indexOf(candidate) === index);
+        for (const candidate of candidates) {
+            const start = output.indexOf(candidate);
+            if (start !== -1) {
+                return output.slice(0, start) + output.slice(start + candidate.length);
+            }
+        }
+        return output;
+    }
+    outputContainsVerificationToken(output, token, injectedTaskText) {
+        if (!token) {
+            return false;
+        }
+        return this.stripInjectedTaskEcho(output, injectedTaskText).includes(token);
+    }
+    prepareInteractiveSpawnTask(agentName, taskText) {
+        if (Buffer.byteLength(taskText, 'utf8') <= WorkflowRunner.PTY_TASK_ARG_SIZE_LIMIT) {
+            return {
+                spawnTaskText: taskText,
+                promptTaskText: taskText,
+            };
+        }
+        const taskTmpDir = mkdtempSync(path.join(tmpdir(), 'relay-pty-task-'));
+        const taskTmpFile = path.join(taskTmpDir, `${agentName}-${Date.now()}.txt`);
+        writeFileSync(taskTmpFile, taskText, { encoding: 'utf8', mode: 0o600, flag: 'wx' });
+        const promptTaskText = `TASK_FILE:${taskTmpFile}\n` +
+            'Read that file completely before taking any action.\n' +
+            'Treat the file contents as the full workflow task and follow them exactly.\n' +
+            'Do not ask for the task again.';
+        return {
+            spawnTaskText: promptTaskText,
+            promptTaskText,
+            taskTmpFile,
+        };
+    }
     firstMeaningfulLine(output) {
         return output
             .split('\n')
@@ -4044,6 +4114,7 @@ export class WorkflowRunner {
             '(b) outputting the exact text "/exit" on its own line as a fallback. ' +
             'Do not wait for further input — terminate immediately after finishing. ' +
             'Do NOT spawn sub-agents unless the task explicitly requires it.';
+        const preparedTask = this.prepareInteractiveSpawnTask(agentName, taskWithExit);
         // Register PTY output listener before spawning so we capture everything
         this.ptyOutputBuffers.set(agentName, []);
         // Open a log file so `agents:logs <name>` works for workflow-spawned agents
@@ -4077,7 +4148,7 @@ export class WorkflowRunner {
                 model: agentDef.constraints?.model,
                 args: interactiveSpawnPolicy.args,
                 channels: agentChannels,
-                task: taskWithExit,
+                task: preparedTask.spawnTaskText,
                 idleThresholdSecs: agentDef.constraints?.idleThresholdSecs,
                 cwd: agentCwd,
             });
@@ -4159,7 +4230,7 @@ export class WorkflowRunner {
             // Register agent handle for hub-mediated nudging
             this.activeAgentHandles.set(agentName, agent);
             // Wait for agent to exit, with idle nudging if configured
-            exitResult = await this.waitForExitWithIdleNudging(agent, agentDef, step, timeoutMs, options.preserveOnIdle ?? this.shouldPreserveIdleSupervisor(agentDef, step, options.evidenceRole));
+            exitResult = await this.waitForExitWithIdleNudging(agent, agentDef, step, timeoutMs, preparedTask.promptTaskText, options.preserveOnIdle ?? this.shouldPreserveIdleSupervisor(agentDef, step, options.evidenceRole));
             // Stop heartbeat now that agent has exited
             stopHeartbeat?.();
             if (exitResult === 'timeout') {
@@ -4169,7 +4240,7 @@ export class WorkflowRunner {
                 let timeoutRecovered = false;
                 if (step.verification) {
                     const ptyOutput = (this.ptyOutputBuffers.get(agentName) ?? []).join('');
-                    const verificationResult = this.runVerification(step.verification, ptyOutput, step.name, undefined, { allowFailure: true });
+                    const verificationResult = this.runVerification(step.verification, ptyOutput, step.name, preparedTask.promptTaskText, { allowFailure: true });
                     if (verificationResult.passed) {
                         this.log(`[${step.name}] Agent timed out but verification passed — treating as complete`);
                         this.postToChannel(`**[${step.name}]** Agent idle after completing work — verification passed, releasing`);
@@ -4216,6 +4287,9 @@ export class WorkflowRunner {
             this.unregisterWorker(agentName);
             this.supervisedRuntimeAgents.delete(agentName);
             this.runtimeStepAgents.delete(agentName);
+            if (preparedTask.taskTmpFile) {
+                await unlink(preparedTask.taskTmpFile).catch(() => undefined);
+            }
         }
         let output;
         if (ptyChunks.length > 0) {
@@ -4243,6 +4317,7 @@ export class WorkflowRunner {
             output,
             exitCode: agent?.exitCode,
             exitSignal: agent?.exitSignal,
+            promptTaskText: preparedTask.promptTaskText,
         };
     }
     // ── Idle nudging ────────────────────────────────────────────────────────
@@ -4288,7 +4363,7 @@ export class WorkflowRunner {
      * Wait for agent exit with idle detection and nudging.
      * If no idle nudge config is set, falls through to simple waitForExit.
      */
-    async waitForExitWithIdleNudging(agent, agentDef, step, timeoutMs, preserveIdleSupervisor = false) {
+    async waitForExitWithIdleNudging(agent, agentDef, step, timeoutMs, promptTaskText, preserveIdleSupervisor = false) {
         const nudgeConfig = this.currentConfig?.swarm.idleNudge;
         if (!nudgeConfig) {
             if (preserveIdleSupervisor) {
@@ -4309,22 +4384,10 @@ export class WorkflowRunner {
                 ]);
                 if (result.kind === 'idle' && result.result === 'idle') {
                     // Check verification before treating idle as complete.
-                    // Mirror runVerification's double-occurrence guard: if the task text
-                    // contains the token (from the prompt instruction), require a second
-                    // occurrence from the agent's actual output to avoid false positives.
                     if (step.verification && step.verification.type === 'output_contains') {
                         const token = step.verification.value;
                         const ptyOutput = (this.ptyOutputBuffers.get(agent.name) ?? []).join('');
-                        const taskText = step.task ?? '';
-                        const taskHasToken = taskText.includes(token);
-                        let verificationPassed = true;
-                        if (taskHasToken) {
-                            const first = ptyOutput.indexOf(token);
-                            verificationPassed = first !== -1 && ptyOutput.includes(token, first + token.length);
-                        }
-                        else {
-                            verificationPassed = ptyOutput.includes(token);
-                        }
+                        const verificationPassed = this.outputContainsVerificationToken(ptyOutput, token, promptTaskText);
                         if (!verificationPassed) {
                             // The broker fires agent_idle only once per idle transition.
                             // If the agent is still working (will produce output then idle again),
@@ -4504,22 +4567,8 @@ export class WorkflowRunner {
         };
         switch (check.type) {
             case 'output_contains': {
-                // Guard against false positives: the PTY captures the injected task text
-                // verbatim, so if the verification token appears in the task itself the
-                // check would pass immediately without the agent doing any real work.
-                // When the task contains the token, require a SECOND occurrence — one
-                // from the task injection and one from the agent's actual response.
                 const token = check.value;
-                const taskHasToken = injectedTaskText ? injectedTaskText.includes(token) : false;
-                if (taskHasToken) {
-                    const first = output.indexOf(token);
-                    const hasSecond = first !== -1 && output.includes(token, first + token.length);
-                    if (!hasSecond) {
-                        return fail(`Verification failed for "${stepName}": output does not contain "${token}" ` +
-                            `(token found only in task injection — agent must output it explicitly)`);
-                    }
-                }
-                else if (!output.includes(token)) {
+                if (!this.outputContainsVerificationToken(output, token, injectedTaskText)) {
                     return fail(`Verification failed for "${stepName}": output does not contain "${token}"`);
                 }
                 break;
@@ -5096,8 +5145,15 @@ export class WorkflowRunner {
             .replace(/-+/g, '-')
             .slice(0, 32);
     }
+    /** Validate that a runId is safe for use in file paths (no traversal). */
+    validateRunId(runId) {
+        if (/[/\\]|^\.\.?$/.test(runId) || runId.includes('..')) {
+            throw new Error(`Invalid runId: "${runId}" contains path traversal characters`);
+        }
+    }
     /** Directory for persisted step outputs: .agent-relay/step-outputs/{runId}/ */
     getStepOutputDir(runId) {
+        this.validateRunId(runId);
         return path.join(this.cwd, '.agent-relay', 'step-outputs', runId);
     }
     /** Persist step output to disk and post full output as a channel message. */
@@ -5182,6 +5238,133 @@ export class WorkflowRunner {
             return undefined;
         }
     }
+    /** Match the best workflow from config given a set of cached step names. */
+    matchWorkflowFromCache(workflows, cachedStepNames) {
+        if (workflows.length === 1)
+            return workflows[0];
+        if (cachedStepNames.size === 0) {
+            // No cached steps to disambiguate — ambiguous when multiple workflows exist
+            this.log('[resume] Multiple workflows in config with empty cache — cannot disambiguate');
+            return null;
+        }
+        // Score each workflow by how many cached steps match, excluding those with unknown steps
+        const scored = workflows
+            .map((candidate) => ({
+            workflow: candidate,
+            matchedSteps: candidate.steps.filter((step) => cachedStepNames.has(step.name)).length,
+            unknownSteps: [...cachedStepNames].filter((name) => !candidate.steps.some((step) => step.name === name)).length,
+        }))
+            .filter((candidate) => candidate.unknownSteps === 0)
+            .sort((a, b) => b.matchedSteps - a.matchedSteps);
+        return scored[0]?.workflow ?? null;
+    }
+    reconstructRunFromCache(runId, config) {
+        const stepOutputDir = this.getStepOutputDir(runId);
+        if (!existsSync(stepOutputDir))
+            return null;
+        let resumeConfig = config ?? this.currentConfig;
+        if (!resumeConfig) {
+            // Attempt to load config from relay.yaml on disk (resume() may call before runWorkflowCore sets currentConfig)
+            const yamlPath = path.join(this.cwd, 'relay.yaml');
+            if (existsSync(yamlPath)) {
+                try {
+                    const raw = readFileSync(yamlPath, 'utf-8');
+                    resumeConfig = this.parseYamlString(raw, yamlPath);
+                }
+                catch {
+                    return null;
+                }
+            }
+            else {
+                return null;
+            }
+        }
+        let entries;
+        try {
+            entries = readdirSync(stepOutputDir, { withFileTypes: true });
+        }
+        catch {
+            return null;
+        }
+        const cachedStepNames = new Set(entries
+            .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+            .map((entry) => entry.name.slice(0, -3))
+            .filter(Boolean));
+        const workflows = resumeConfig.workflows ?? [];
+        if (workflows.length === 0)
+            return null;
+        // Empty cache directory is valid — all steps will be re-run
+        const workflow = this.matchWorkflowFromCache(workflows, cachedStepNames);
+        if (!workflow)
+            return null;
+        // Use actual file modification times from cached outputs instead of synthetic timestamps
+        const stepMtimes = new Map();
+        let earliestMtime = Date.now();
+        for (const stepName of cachedStepNames) {
+            try {
+                const mdPath = path.join(stepOutputDir, `${stepName}.md`);
+                const reportPath = path.join(stepOutputDir, `${stepName}.report.json`);
+                const mdStat = existsSync(mdPath) ? statSync(mdPath) : null;
+                const reportStat = existsSync(reportPath) ? statSync(reportPath) : null;
+                // Use the latest mtime between .md and .report.json
+                const mtime = Math.max(mdStat?.mtimeMs ?? 0, reportStat?.mtimeMs ?? 0);
+                if (mtime > 0) {
+                    stepMtimes.set(stepName, new Date(mtime).toISOString());
+                    if (mtime < earliestMtime)
+                        earliestMtime = mtime;
+                }
+            }
+            catch {
+                // Fall back to current time if stat fails
+            }
+        }
+        const fallbackTime = new Date().toISOString();
+        const completedSteps = new Set(workflow.steps.filter((step) => cachedStepNames.has(step.name)).map((step) => step.name));
+        // Heuristic: mark the first eligible non-completed step as failed (the likely failure point)
+        const failedStepName = workflow.steps.find((step) => !completedSteps.has(step.name) && (step.dependsOn ?? []).every((dep) => completedSteps.has(dep)))?.name;
+        const runStartedAt = new Date(earliestMtime).toISOString();
+        const run = {
+            id: runId,
+            workspaceId: this.workspaceId,
+            workflowName: workflow.name,
+            pattern: resumeConfig.swarm.pattern,
+            status: 'failed',
+            config: resumeConfig,
+            startedAt: runStartedAt,
+            createdAt: runStartedAt,
+            updatedAt: fallbackTime,
+        };
+        const stepStates = new Map();
+        for (const step of workflow.steps) {
+            const isNonAgent = step.type === 'deterministic' || step.type === 'worktree' || step.type === 'integration';
+            const cachedOutput = completedSteps.has(step.name) ? this.loadStepOutput(runId, step.name) : undefined;
+            const status = completedSteps.has(step.name) ? 'completed' : step.name === failedStepName ? 'failed' : 'pending';
+            const stepRow = {
+                id: this.generateId(),
+                runId,
+                stepName: step.name,
+                agentName: isNonAgent ? null : (step.agent ?? null),
+                stepType: isNonAgent ? step.type : 'agent',
+                status,
+                task: step.type === 'deterministic'
+                    ? (step.command ?? '')
+                    : step.type === 'worktree'
+                        ? (step.branch ?? '')
+                        : step.type === 'integration'
+                            ? (`${step.integration}.${step.action}`)
+                            : (step.task ?? ''),
+                dependsOn: step.dependsOn ?? [],
+                output: cachedOutput,
+                error: status === 'failed' ? 'Recovered from cached step outputs' : undefined,
+                completedAt: status === 'completed' ? (stepMtimes.get(step.name) ?? fallbackTime) : undefined,
+                retryCount: 0,
+                createdAt: stepMtimes.get(step.name) ?? fallbackTime,
+                updatedAt: stepMtimes.get(step.name) ?? fallbackTime,
+            };
+            stepStates.set(step.name, { row: stepRow });
+        }
+        return { run, stepStates };
+    }
     /** Get or create the worker logs directory (.agent-relay/team/worker-logs) */
     getWorkerLogsDir() {
         const logsDir = path.join(this.cwd, '.agent-relay', 'team', 'worker-logs');