palmier 0.2.5 → 0.2.6

package/src/commands/run.ts

@@ -1,15 +1,17 @@
 import * as fs from "fs";
 import * as path from "path";
-import { spawnCommand } from "../spawn-command.js";
+import * as readline from "readline";
+import { spawnCommand, spawnStreamingCommand } from "../spawn-command.js";
 import { loadConfig } from "../config.js";
 import { connectNats } from "../nats-client.js";
 import { parseTaskFile, getTaskDir, writeTaskFile, writeTaskStatus, readTaskStatus, appendHistory } from "../task.js";
 import { getAgent } from "../agents/agent.js";
 import { getPlatform } from "../platform/index.js";
-import { TASK_SUCCESS_MARKER, TASK_FAILURE_MARKER, TASK_REPORT_PREFIX, TASK_PERMISSION_PREFIX } from "../agents/shared-prompt.js";
+import { TASK_SUCCESS_MARKER, TASK_FAILURE_MARKER, TASK_REPORT_PREFIX, TASK_PERMISSION_PREFIX, TASK_INPUT_PREFIX } from "../agents/shared-prompt.js";
+import type { AgentTool } from "../agents/agent.js";
+import { publishHostEvent } from "../events.js";
 import type { HostConfig, ParsedTask, TaskRunningState, RequiredPermission } from "../types.js";
 import type { NatsConnection } from "nats";
-import { StringCodec } from "nats";
 
 /**
  * Write a time-stamped RESULT file with frontmatter.
@@ -34,6 +36,113 @@ function writeResult(
   return resultFileName;
 }
 
+/**
+ * Shared context for agent invocation retry loops.
+ * Passed around to avoid threading many individual parameters.
+ */
+interface InvocationContext {
+  agent: AgentTool;
+  task: ParsedTask;
+  taskDir: string;
+  guiEnv: Record<string, string>;
+  nc: NatsConnection | undefined;
+  config: HostConfig;
+  taskId: string;
+  useHttp: boolean;
+  /** Mutable — accumulates across invocations within a run. */
+  transientPermissions: RequiredPermission[];
+}
+
+interface InvocationResult {
+  output: string;
+  outcome: TaskRunningState;
+  reportFiles: string[];
+  requiredPermissions: RequiredPermission[];
+}
+
+/**
+ * Invoke the agent CLI with a retry loop for permissions and user input.
+ *
+ * Both standard and command-triggered execution use this.
+ * The `invokeTask` is the ParsedTask whose prompt is passed to the agent
+ * (for command-triggered mode this is the per-line augmented task).
+ */
+async function invokeAgentWithRetry(
+  ctx: InvocationContext,
+  invokeTask: ParsedTask,
+): Promise<InvocationResult> {
+  let retryPrompt: string | undefined;
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const { command, args, stdin } = ctx.agent.getTaskRunCommandLine(invokeTask, retryPrompt, ctx.transientPermissions);
+    const result = await spawnCommand(command, args, {
+      cwd: ctx.taskDir,
+      env: { ...ctx.guiEnv, PALMIER_TASK_ID: ctx.task.frontmatter.id },
+      echoStdout: true,
+      resolveOnFailure: true,
+      stdin,
+    });
+
+    const outcome: TaskRunningState = result.exitCode !== 0 ? "failed" : parseTaskOutcome(result.output);
+    const reportFiles = parseReportFiles(result.output);
+    const requiredPermissions = parsePermissions(result.output);
+
+    // Permission retry
+    if (outcome === "failed" && requiredPermissions.length > 0) {
+      const response = await requestPermission(ctx.nc, ctx.config, ctx.task, ctx.taskDir, requiredPermissions, ctx.useHttp);
+      await publishPermissionResolved(ctx.nc, ctx.config, ctx.taskId, response, ctx.useHttp);
+
+      if (response === "aborted") {
+        return { output: result.output, outcome: "failed", reportFiles, requiredPermissions };
+      }
+
+      const newPerms = requiredPermissions.filter(
+        (rp) => !ctx.task.frontmatter.permissions?.some((ep) => ep.name === rp.name)
+          && !ctx.transientPermissions.some((ep) => ep.name === rp.name),
+      );
+
+      if (response === "granted_all") {
+        ctx.task.frontmatter.permissions = [...(ctx.task.frontmatter.permissions ?? []), ...newPerms];
+        invokeTask.frontmatter.permissions = ctx.task.frontmatter.permissions;
+        writeTaskFile(ctx.taskDir, ctx.task);
+      } else {
+        ctx.transientPermissions = [...ctx.transientPermissions, ...newPerms];
+      }
+
+      retryPrompt = "Permissions granted, please continue.";
+      continue;
+    }
+
+    // Input retry
+    const inputRequests = parseInputRequests(result.output);
+    if (outcome === "failed" && inputRequests.length > 0) {
+      const response = await requestUserInput(ctx.nc, ctx.config, ctx.task, ctx.taskDir, inputRequests, ctx.useHttp);
+      await publishInputResolved(ctx.nc, ctx.config, ctx.taskId, response === "aborted" ? "aborted" : "provided", ctx.useHttp);
+
+      if (response === "aborted") {
+        return { output: result.output, outcome: "failed", reportFiles, requiredPermissions };
+      }
+
+      const inputLines = inputRequests.map((desc, i) => `- ${desc} → ${response[i]}`).join("\n");
+      retryPrompt = `The user provided the following inputs:\n${inputLines}\nPlease continue with these values.`;
+      continue;
+    }
+
+    // Normal completion (success or non-retryable failure)
+    return { output: result.output, outcome, reportFiles, requiredPermissions };
+  }
+}
+
+/**
+ * If the RPC handler already wrote "aborted" to status.json (e.g. via task.abort),
+ * respect that instead of overwriting with the process's own outcome.
+ */
+function resolveOutcome(taskDir: string, outcome: TaskRunningState): TaskRunningState {
+  const current = readTaskStatus(taskDir);
+  if (current?.running_state === "aborted") return "aborted";
+  return outcome;
+}
+
 /**
  * Execute a task by ID.
  */
@@ -61,26 +170,13 @@ export async function runCommand(taskId: string): Promise<void> {
     }
   };
 
-  // Handle signals
-  const onSignal = async () => {
-    console.log("Received signal, cleaning up...");
-    const endTime = Date.now();
-    const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, "abort", startTime, endTime, "", [], []);
-    appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
-    await publishTaskEvent(nc, config, taskDir, taskId, "abort", useHttp, taskName);
-    await cleanup();
-    process.exit(1);
-  };
-  process.on("SIGINT", onSignal);
-  process.on("SIGTERM", onSignal);
-
   try {
     if (useNats) {
       nc = await connectNats(config);
     }
 
     // Mark as started immediately
-    await publishTaskEvent(nc, config, taskDir, taskId, "start", useHttp, taskName);
+    await publishTaskEvent(nc, config, taskDir, taskId, "started", useHttp, taskName);
 
     // If requires_confirmation, notify clients and wait
     if (task.frontmatter.requires_confirmation) {
@@ -90,130 +186,213 @@ export async function runCommand(taskId: string): Promise<void> {
       if (!confirmed) {
         console.log("Task aborted by user.");
         const endTime = Date.now();
-        const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, "abort", startTime, endTime, "", [], []);
+        const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, "aborted", startTime, endTime, "", [], []);
         appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
-        await publishTaskEvent(nc, config, taskDir, taskId, "abort", useHttp, taskName);
+        await publishTaskEvent(nc, config, taskDir, taskId, "aborted", useHttp, taskName);
         await cleanup();
         return;
       }
       console.log("Task confirmed by user.");
     }
 
-    // Execution loop: retry on permission failure if user grants
+    // Shared invocation context
     const guiEnv = getPlatform().getGuiEnv();
-
     const agent = getAgent(task.frontmatter.agent);
-
-    let lastOutput = "";
-    let lastOutcome: TaskRunningState = "fail";
-    let lastReportFiles: string[] = [];
-    let lastRequiredPermissions: RequiredPermission[] = [];
-    let lastEndTime = Date.now();
-
-    let retryPrompt: string | undefined;
-    let transientPermissions: RequiredPermission[] = [];
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      const { command, args } = agent.getTaskRunCommandLine(task, retryPrompt, transientPermissions);
-      const result = await spawnCommand(command, args, {
-        cwd: taskDir,
-        env: {
-          ...guiEnv,
-          PALMIER_TASK_ID: task.frontmatter.id,
-        },
-        echoStdout: true,
-        forwardSignals: true,
-        resolveOnFailure: true,
-      });
-      lastOutput = result.output;
-      lastEndTime = Date.now();
-
-      lastOutcome = result.exitCode !== 0 ? "fail" : parseTaskOutcome(lastOutput);
-      lastReportFiles = parseReportFiles(lastOutput);
-      lastRequiredPermissions = parsePermissions(lastOutput);
-
-      // If failed with permission requirements, ask user to grant
-      if (lastOutcome === "fail" && lastRequiredPermissions.length > 0) {
-        const response = await requestPermission(nc, config, task, taskDir, lastRequiredPermissions, useHttp);
-        await publishPermissionResolved(nc, config, taskId, response, useHttp);
-
-        if (response === "aborted") {
-          console.log("Permission request aborted by user.");
-          break;
-        }
-
-        const newPerms = lastRequiredPermissions.filter(
-          (rp) => !task.frontmatter.permissions?.some((ep) => ep.name === rp.name)
-            && !transientPermissions.some((ep) => ep.name === rp.name),
-        );
-
-        if (response === "granted_all") {
-          // Persist permissions to task frontmatter for all future runs
-          task.frontmatter.permissions = [...(task.frontmatter.permissions ?? []), ...newPerms];
-          writeTaskFile(taskDir, task);
-        } else {
-          // "granted" — allow for this run only
-          transientPermissions = [...transientPermissions, ...newPerms];
-        }
-
-        console.log(`Permissions granted, retrying task ${taskId}...`);
-        retryPrompt = "Permissions granted, please continue.";
-        continue;
+    const ctx: InvocationContext = {
+      agent, task, taskDir, guiEnv, nc, config, taskId, useHttp,
+      transientPermissions: [],
+    };
+
+    if (task.frontmatter.command) {
+      // Command-triggered mode
+      const result = await runCommandTriggeredMode(ctx);
+      const outcome = resolveOutcome(taskDir, result.outcome);
+      const resultFileName = writeResult(
+        taskDir, taskName, taskSnapshotName, outcome,
+        startTime, result.endTime, result.output, [], [],
+      );
+      appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
+      await publishTaskEvent(nc, config, taskDir, taskId, outcome, useHttp, taskName);
+      console.log(`Task ${taskId} completed (command-triggered).`);
+    } else {
+      // Standard execution
+      const result = await invokeAgentWithRetry(ctx, task);
+      const outcome = resolveOutcome(taskDir, result.outcome);
+
+      const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, outcome, startTime, Date.now(), result.output, result.reportFiles, result.requiredPermissions);
+      appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
+      await publishTaskEvent(nc, config, taskDir, taskId, outcome, useHttp, taskName);
+
+      if (result.reportFiles.length > 0) {
+        await publishHostEvent(nc, config, taskId, {
+          event_type: "report-generated",
+          name: taskName,
+          report_files: result.reportFiles,
+          running_state: outcome,
+        }, useHttp);
       }
 
-      // Normal completion (success or non-permission failure)
-      break;
+      console.log(`Task ${taskId} completed.`);
     }
-
-    // Write result and history once after the loop
-    const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, lastOutcome, startTime, lastEndTime, lastOutput, lastReportFiles, lastRequiredPermissions);
-    appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
-    await publishTaskEvent(nc, config, taskDir, taskId, lastOutcome, useHttp, taskName);
-    console.log(`Task ${taskId} completed.`);
   } catch (err) {
     console.error(`Task ${taskId} failed:`, err);
     const endTime = Date.now();
+    const outcome = resolveOutcome(taskDir, "failed");
     const errorMsg = err instanceof Error ? err.message : String(err);
-    const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, "fail", startTime, endTime, errorMsg, [], []);
+    const resultFileName = writeResult(taskDir, taskName, taskSnapshotName, outcome, startTime, endTime, errorMsg, [], []);
     appendHistory(config.projectRoot, { task_id: taskId, result_file: resultFileName });
-    await publishTaskEvent(nc, config, taskDir, taskId, "fail", useHttp, taskName);
+    await publishTaskEvent(nc, config, taskDir, taskId, outcome, useHttp, taskName);
     process.exitCode = 1;
   } finally {
     await cleanup();
   }
 }
 
-const sc = StringCodec();
+const MAX_QUEUE_SIZE = 100;
+const MAX_LOG_ENTRIES = 1000;
+/** Max input line length (chars). Long emails can take up to 200k chars. */
+const MAX_LINE_LENGTH = 200_000;
 
 /**
- * Write status.json and notify connected clients via NATS and/or HTTP SSE.
+ * Command-triggered execution mode.
+ *
+ * Spawns a long-running shell command and, for each line of stdout,
+ * invokes the agent CLI with the user's prompt augmented by that line.
+ * Processes lines sequentially with a bounded queue.
  */
-async function publishHostEvent(
-  nc: NatsConnection | undefined,
-  config: HostConfig,
-  taskId: string,
-  payload: Record<string, unknown>,
-  useHttp: boolean,
-): Promise<void> {
-  const subject = `host-event.${config.hostId}.${taskId}`;
+async function runCommandTriggeredMode(
+  ctx: InvocationContext,
+): Promise<{ outcome: TaskRunningState; endTime: number; output: string }> {
+  const commandStr = ctx.task.frontmatter.command!;
+  console.log(`[command-triggered] Spawning: ${commandStr}`);
+
+  const child = spawnStreamingCommand(commandStr, {
+    cwd: ctx.taskDir,
+    env: { ...ctx.guiEnv, PALMIER_TASK_ID: ctx.task.frontmatter.id },
+  });
+
+  // Stats
+  let linesProcessed = 0;
+  let invocationsSucceeded = 0;
+  let invocationsFailed = 0;
+
+  // Bounded queue for incoming lines
+  const lineQueue: string[] = [];
+  let processing = false;
+  let commandExited = false;
+  let resolveWhenDone: (() => void) | undefined;
+
+  // Rolling log of per-line agent outputs
+  const logPath = path.join(ctx.taskDir, "command-output.log");
+  function appendLog(line: string, agentOutput: string, outcome: string) {
+    const entry = `[${new Date().toISOString()}] (${outcome}) input: ${line}\n${agentOutput}\n---\n`;
+    fs.appendFileSync(logPath, entry, "utf-8");
+
+    // Trim log if too large (keep last MAX_LOG_ENTRIES entries)
+    try {
+      const content = fs.readFileSync(logPath, "utf-8");
+      const entries = content.split("\n---\n").filter(Boolean);
+      if (entries.length > MAX_LOG_ENTRIES) {
+        const trimmed = entries.slice(-MAX_LOG_ENTRIES).join("\n---\n") + "\n---\n";
+        fs.writeFileSync(logPath, trimmed, "utf-8");
+      }
+    } catch { /* ignore trim errors */ }
+  }
 
-  if (nc) {
-    nc.publish(subject, sc.encode(JSON.stringify(payload)));
-    console.log(`[nats] ${subject} →`, payload);
+  async function processLine(line: string): Promise<void> {
+    linesProcessed++;
+    if (line.length > MAX_LINE_LENGTH) {
+      console.warn(`[command-triggered] Skipping line #${linesProcessed}: ${line.length} chars exceeds limit`);
+      invocationsFailed++;
+      appendLog(line.slice(0, 200) + "...(truncated)", "", "skipped");
+      return;
+    }
+    console.log(`[command-triggered] Processing line #${linesProcessed}: ${line}`);
+
+    const perLinePrompt = `${ctx.task.frontmatter.user_prompt}\n\nProcess this input:\n${line}`;
+    const perLineTask: ParsedTask = {
+      frontmatter: { ...ctx.task.frontmatter, user_prompt: perLinePrompt },
+      body: "",
+    };
+
+    const result = await invokeAgentWithRetry(ctx, perLineTask);
+    if (result.outcome === "finished") {
+      invocationsSucceeded++;
+    } else {
+      invocationsFailed++;
+    }
+    appendLog(line, result.output, result.outcome);
   }
 
-  if (useHttp && config.directPort) {
+  async function drainQueue(): Promise<void> {
+    if (processing) return;
+    processing = true;
     try {
-      await fetch(`http://localhost:${config.directPort}/internal/event`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ task_id: taskId, ...payload }),
-      });
-      console.log(`[http] host-event: ${taskId} →`, payload);
-    } catch (err) {
-      console.error(`[http] Failed to push event:`, err);
+      while (lineQueue.length > 0) {
+        const line = lineQueue.shift()!;
+        await processLine(line);
+      }
+    } finally {
+      processing = false;
+      if (commandExited && lineQueue.length === 0 && resolveWhenDone) {
+        resolveWhenDone();
+      }
     }
   }
+
+  // Read stdout line by line
+  const rl = readline.createInterface({ input: child.stdout! });
+  rl.on("line", (line: string) => {
+    if (!line.trim()) return; // skip empty lines
+    if (lineQueue.length >= MAX_QUEUE_SIZE) {
+      console.warn(`[command-triggered] Queue full, dropping oldest line.`);
+      lineQueue.shift();
+    }
+    lineQueue.push(line);
+    drainQueue().catch((err) => {
+      console.error(`[command-triggered] Error processing line:`, err);
+      invocationsFailed++;
+    });
+  });
+
+  // Log stderr
+  child.stderr?.on("data", (d: Buffer) => process.stderr.write(d));
+
+  // Wait for command to exit
+  const exitCode = await new Promise<number | null>((resolve) => {
+    child.on("close", (code: number | null) => {
+      commandExited = true;
+      rl.close();
+      resolve(code);
+    });
+    child.on("error", (err: Error) => {
+      console.error(`[command-triggered] Command error:`, err);
+      commandExited = true;
+      rl.close();
+      resolve(1);
+    });
+  });
+
+  // Wait for any remaining queued lines to finish processing
+  if (lineQueue.length > 0 || processing) {
+    await new Promise<void>((resolve) => {
+      resolveWhenDone = resolve;
+      drainQueue();
+    });
+  }
+
+  const endTime = Date.now();
+  const summary = [
+    `Command: ${commandStr}`,
+    `Exit code: ${exitCode}`,
+    `Lines processed: ${linesProcessed}`,
+    `Agent invocations succeeded: ${invocationsSucceeded}`,
+    `Agent invocations failed: ${invocationsFailed}`,
+  ].join("\n");
+
+  // Command-triggered tasks run until the command exits — any exit is a normal finish.
+  const outcome: TaskRunningState = "finished";
+  return { outcome, endTime, output: summary };
 }
 
 async function publishTaskEvent(
@@ -276,11 +455,11 @@ async function requestPermission(
   return new Promise<"granted" | "granted_all" | "aborted">((resolve) => {
     const watcher = fs.watch(statusPath, () => {
       const status = readTaskStatus(taskDir);
-      if (!status || status.user_input === undefined) return;
+      if (!status || !status.user_input?.length) return;
       watcher.close();
-      const response = status.user_input as "granted" | "granted_all" | "aborted";
+      const response = status.user_input[0] as "granted" | "granted_all" | "aborted";
       writeTaskStatus(taskDir, {
-        running_state: response === "aborted" ? "abort" : "start",
+        running_state: response === "aborted" ? "aborted" : "started",
         time_stamp: Date.now(),
       });
       resolve(response);
@@ -302,6 +481,58 @@ async function publishPermissionResolved(
   }, useHttp);
 }
 
+async function requestUserInput(
+  nc: NatsConnection | undefined,
+  config: HostConfig,
+  task: ParsedTask,
+  taskDir: string,
+  inputDescriptions: string[],
+  useHttp: boolean,
+): Promise<string[] | "aborted"> {
+  const taskId = task.frontmatter.id;
+  const statusPath = path.join(taskDir, "status.json");
+
+  const currentStatus = readTaskStatus(taskDir)!;
+  writeTaskStatus(taskDir, { ...currentStatus, pending_input: inputDescriptions });
+
+  await publishHostEvent(nc, config, taskId, {
+    event_type: "input-request",
+    host_id: config.hostId,
+    input_descriptions: inputDescriptions,
+    name: task.frontmatter.name,
+  }, useHttp);
+
+  return new Promise<string[] | "aborted">((resolve) => {
+    const watcher = fs.watch(statusPath, () => {
+      const status = readTaskStatus(taskDir);
+      if (!status || !status.user_input?.length) return;
+      watcher.close();
+      const response = status.user_input;
+      if (response.length === 1 && response[0] === "aborted") {
+        writeTaskStatus(taskDir, { running_state: "aborted", time_stamp: Date.now() });
+        resolve("aborted");
+      } else {
+        writeTaskStatus(taskDir, { running_state: "started", time_stamp: Date.now() });
+        resolve(response);
+      }
+    });
+  });
+}
+
+async function publishInputResolved(
+  nc: NatsConnection | undefined,
+  config: HostConfig,
+  taskId: string,
+  status: "provided" | "aborted",
+  useHttp: boolean,
+): Promise<void> {
+  await publishHostEvent(nc, config, taskId, {
+    event_type: "input-resolved",
+    host_id: config.hostId,
+    status,
+  }, useHttp);
+}
+
 async function requestConfirmation(
   nc: NatsConnection | undefined,
   config: HostConfig,
@@ -326,12 +557,12 @@ async function requestConfirmation(
   return new Promise<boolean>((resolve) => {
     const watcher = fs.watch(statusPath, () => {
       const status = readTaskStatus(taskDir);
-      if (!status || status.user_input === undefined) return; // still pending
+      if (!status || !status.user_input?.length) return; // still pending
       watcher.close();
-      const confirmed = status.user_input === "confirmed";
+      const confirmed = status.user_input[0] === "confirmed";
       // Clear pending_confirmation/user_input and update running_state
       writeTaskStatus(taskDir, {
-        running_state: confirmed ? "start" : "abort",
+        running_state: confirmed ? "started" : "aborted",
         time_stamp: Date.now(),
       });
       resolve(confirmed);
@@ -374,14 +605,29 @@ function parsePermissions(output: string): RequiredPermission[] {
   return perms;
 }
 
+/**
+ * Extract user input requests from agent output.
+ * Looks for lines matching: [PALMIER_INPUT] <description>
+ */
+function parseInputRequests(output: string): string[] {
+  const regex = new RegExp(`^\\${TASK_INPUT_PREFIX}\\s+(.+)$`, "gm");
+  const inputs: string[] = [];
+  let match;
+  while ((match = regex.exec(output)) !== null) {
+    const desc = match[1].trim();
+    if (desc) inputs.push(desc);
+  }
+  return inputs;
+}
+
 /**
  * Parse the agent's output for success/failure markers.
- * Falls back to "finish" if no marker is found.
+ * Falls back to "finished" if no marker is found.
  */
 function parseTaskOutcome(output: string): TaskRunningState {
   const lastChunk = output.slice(-500);
-  if (lastChunk.includes(TASK_FAILURE_MARKER)) return "fail";
-  if (lastChunk.includes(TASK_SUCCESS_MARKER)) return "finish";
-  return "finish";
+  if (lastChunk.includes(TASK_FAILURE_MARKER)) return "failed";
+  if (lastChunk.includes(TASK_SUCCESS_MARKER)) return "finished";
+  return "finished";
 }