opencode-supertask 0.1.40 → 0.1.41

package/dist/worker/index.d.ts

@@ -3,6 +3,8 @@ import { G as GatewayConfig } from '../config-CCUuD6Az.js';
 interface WorkerEngineOptions {
     opencodeBin?: string;
     maxOutputChars?: number;
+    settlementRetryDelaysMs?: number[];
+    settlementRetryIntervalMs?: number;
 }
 interface InterruptedTaskRun {
     taskId: number;
@@ -17,9 +19,13 @@ declare class WorkerEngine {
     private pollTimer;
     private heartbeatTimer;
     private pollCyclePromise;
+    private shutdownDeadlineMs;
+    private settlementRetryWakeups;
     private cfg;
     private opencodeBin;
     private maxOutputChars;
+    private settlementRetryDelaysMs;
+    private settlementRetryIntervalMs;
     constructor(cfg: GatewayConfig, options?: WorkerEngineOptions);
     start(): void;
     stop(gracePeriodMs?: number): Promise<InterruptedTaskRun[]>;
@@ -32,6 +38,8 @@ declare class WorkerEngine {
     private spawnTask;
     private handleChildClose;
     private settleEntry;
+    private commitEntryWithRetry;
+    private waitForSettlementRetry;
     private commitEntry;
     private reconcileCancelledTasks;
     private reconcileQuarantinedTasks;

package/dist/worker/index.js

@@ -5538,13 +5538,23 @@ var TaskService = class {
       });
       const maxRetries = normalizedData.maxRetries ?? task.maxRetries ?? 3;
       const exhausted = task.status === "failed" && (task.retryCount ?? 0) > maxRetries;
-      return tx.update(tasks2).set({
+      const updated = tx.update(tasks2).set({
         ...normalizedData,
         ...exhausted ? {
           status: "dead_letter",
           retryAfter: null
         } : {}
       }).where(eq(tasks2.id, id)).returning().get() ?? null;
+      if (exhausted && updated) {
+        const finishedAt = /* @__PURE__ */ new Date();
+        tx.update(tasks2).set({
+          status: "dead_letter",
+          finishedAt,
+          retryAfter: null,
+          resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
+        }).where(blockedDependentsOf(id)).run();
+      }
+      return updated;
     }, { behavior: "immediate" });
   }
   static validateNewTask(data) {
@@ -5565,7 +5575,7 @@ var TaskService = class {
       throw new Error(`${name} \u5FC5\u987B\u662F ${min} \u5230 ${max} \u4E4B\u95F4\u7684\u6574\u6570`);
     }
   }
-  static async next(scope = {}) {
+  static buildRunnableTaskWhere(scope) {
     const baseConditions = [...this.buildScopeWhere(scope)];
     const nowMs = Date.now();
     const retryAfterFilter = or(
@@ -5600,40 +5610,43 @@ var TaskService = class {
     if (batchFilter) {
       conditions.push(batchFilter);
     }
-    const result = await db.select().from(tasks2).where(and(
+    return and(
       ...conditions,
       or(
         isNull(tasks2.dependsOn),
         sql`EXISTS (
-                        SELECT 1 FROM tasks AS dependency_task
-                        WHERE dependency_task.id = ${tasks2.dependsOn}
-                          AND dependency_task.status = 'done'
-                          AND dependency_task.cwd IS ${tasks2.cwd}
-                    )`
+                    SELECT 1 FROM tasks AS dependency_task
+                    WHERE dependency_task.id = ${tasks2.dependsOn}
+                      AND dependency_task.status = 'done'
+                      AND dependency_task.cwd IS ${tasks2.cwd}
+                )`
       ),
       or(
         isNull(tasks2.batchId),
         sql`trim(${tasks2.batchId}, ${TASK_BATCH_TRIM_CHARACTERS}) = ''`,
         sql`NOT EXISTS (
-                        SELECT 1 FROM tasks AS running_batch_task
-                        WHERE trim(running_batch_task.batch_id, ${TASK_BATCH_TRIM_CHARACTERS})
-                            = trim(${tasks2.batchId}, ${TASK_BATCH_TRIM_CHARACTERS})
-                          AND (
-                              running_batch_task.status = 'running'
-                              OR EXISTS (
-                                  SELECT 1 FROM task_runs AS running_batch_run
-                                  WHERE running_batch_run.task_id = running_batch_task.id
-                                    AND running_batch_run.status = 'running'
-                              )
+                    SELECT 1 FROM tasks AS running_batch_task
+                    WHERE trim(running_batch_task.batch_id, ${TASK_BATCH_TRIM_CHARACTERS})
+                        = trim(${tasks2.batchId}, ${TASK_BATCH_TRIM_CHARACTERS})
+                      AND (
+                          running_batch_task.status = 'running'
+                          OR EXISTS (
+                              SELECT 1 FROM task_runs AS running_batch_run
+                              WHERE running_batch_run.task_id = running_batch_task.id
+                                AND running_batch_run.status = 'running'
                           )
-                        )`
+                      )
+                    )`
       ),
       sql`NOT EXISTS (
-                    SELECT 1 FROM task_runs AS candidate_active_run
-                    WHERE candidate_active_run.task_id = ${tasks2.id}
-                      AND candidate_active_run.status = 'running'
-                )`
-    )).orderBy(
+                SELECT 1 FROM task_runs AS candidate_active_run
+                WHERE candidate_active_run.task_id = ${tasks2.id}
+                  AND candidate_active_run.status = 'running'
+            )`
+    );
+  }
+  static async next(scope = {}) {
+    const result = await db.select().from(tasks2).where(this.buildRunnableTaskWhere(scope)).orderBy(
       desc(tasks2.urgency),
       desc(tasks2.importance),
       asc(tasks2.createdAt),
@@ -5641,6 +5654,22 @@ var TaskService = class {
     ).limit(1);
     return result[0] ?? null;
   }
+  static async claimNext(scope = {}) {
+    return db.transaction((tx) => {
+      const candidate = tx.select().from(tasks2).where(this.buildRunnableTaskWhere(scope)).orderBy(
+        desc(tasks2.urgency),
+        desc(tasks2.importance),
+        asc(tasks2.createdAt),
+        asc(tasks2.id)
+      ).limit(1).get();
+      if (!candidate) return null;
+      return tx.update(tasks2).set({
+        status: "running",
+        startedAt: /* @__PURE__ */ new Date(),
+        finishedAt: null
+      }).where(eq(tasks2.id, candidate.id)).returning().get() ?? null;
+    }, { behavior: "immediate" });
+  }
   static async countRunning(scope = {}) {
     const scopeConditions = scope.legacyCwd ? [sql`${tasks2.cwd} IS NULL OR trim(${tasks2.cwd}) = ''`] : this.buildScopeWhere(scope);
     if (scope.batchId !== void 0) {
@@ -6273,7 +6302,6 @@ var TaskService = class {
 };
 
 // src/core/process-control.ts
-import { spawnSync } from "child_process";
 import { fileURLToPath as fileURLToPath2 } from "url";
 import { basename, dirname as dirname2, resolve } from "path";
 
@@ -6298,6 +6326,84 @@ function isMatchingDrainProof(message, launchIdentity) {
 
 // src/core/process-control.ts
 var OS_COMMAND_TIMEOUT_MS = 2e3;
+var OS_COMMAND_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+var OS_COMMAND_RESULT_MARKER = "\n\0supertask-os-command-result:";
+var BOUNDED_OS_COMMAND_RUNNER = `
+const { writeSync } = await import('fs');
+let child;
+try {
+    child = Bun.spawn(process.argv.slice(2), {
+        stdin: 'ignore',
+        stdout: 'pipe',
+        stderr: 'ignore',
+    });
+} catch {
+    process.exit(127);
+}
+const terminateGroup = () => {
+    if (process.platform !== 'win32') {
+        try {
+            process.kill(-process.pid, 'SIGKILL');
+        } catch {
+        }
+    }
+    try {
+        child.kill('SIGKILL');
+    } catch {
+    }
+    process.exit(1);
+};
+const timer = setTimeout(terminateGroup, ${OS_COMMAND_TIMEOUT_MS});
+try {
+    const chunks = [];
+    let outputBytes = 0;
+    const readOutput = async () => {
+        const reader = child.stdout.getReader();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) return;
+            outputBytes += value.byteLength;
+            if (outputBytes > ${OS_COMMAND_MAX_OUTPUT_BYTES}) terminateGroup();
+            chunks.push(value);
+        }
+    };
+    const [exitCode] = await Promise.all([child.exited, readOutput()]);
+    clearTimeout(timer);
+    for (const chunk of chunks) writeSync(1, chunk);
+    writeSync(1, ${JSON.stringify(OS_COMMAND_RESULT_MARKER)} + String(exitCode));
+    if (process.platform !== 'win32') terminateGroup();
+    process.exit(0);
+} catch {
+    clearTimeout(timer);
+    terminateGroup();
+}
+`;
+function runBoundedOsCommand(command, args, captureOutput = true) {
+  const result = Bun.spawnSync({
+    cmd: [
+      process.execPath,
+      "-e",
+      BOUNDED_OS_COMMAND_RUNNER,
+      "supertask-os-command",
+      command,
+      ...args
+    ],
+    detached: process.platform !== "win32",
+    maxBuffer: OS_COMMAND_MAX_OUTPUT_BYTES + 1024,
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "ignore"
+  });
+  const output = new TextDecoder().decode(result.stdout);
+  const markerIndex = output.lastIndexOf(OS_COMMAND_RESULT_MARKER);
+  if (markerIndex < 0) return { status: null, stdout: "" };
+  const statusText = output.slice(markerIndex + OS_COMMAND_RESULT_MARKER.length);
+  if (!/^\d+$/.test(statusText)) return { status: null, stdout: "" };
+  return {
+    status: Number(statusText),
+    stdout: captureOutput ? output.slice(0, markerIndex) : ""
+  };
+}
 function isSafePid(pid) {
   return Number.isInteger(pid) && pid > 1 && pid !== process.pid;
 }
@@ -6331,12 +6437,10 @@ function inspectSpawnedProcessTreePresence(pid) {
   }
 }
 function inspectUnixProcess(pid) {
-  const result = spawnSync("ps", ["-o", "pgid=", "-o", "command=", "-p", String(pid)], {
-    encoding: "utf8",
-    stdio: ["ignore", "pipe", "ignore"],
-    timeout: OS_COMMAND_TIMEOUT_MS,
-    killSignal: "SIGKILL"
-  });
+  const result = runBoundedOsCommand(
+    "ps",
+    ["-o", "pgid=", "-o", "command=", "-p", String(pid)]
+  );
   if (result.status !== 0) return null;
   const match = result.stdout.trim().match(/^(\d+)\s+(.+)$/s);
   if (!match) return null;
@@ -6344,30 +6448,18 @@ function inspectUnixProcess(pid) {
 }
 function inspectWindowsCommand(pid) {
   const script = `(Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}").CommandLine`;
-  const result = spawnSync(
+  const result = runBoundedOsCommand(
     "powershell.exe",
-    ["-NoProfile", "-NonInteractive", "-Command", script],
-    {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-      timeout: OS_COMMAND_TIMEOUT_MS,
-      killSignal: "SIGKILL"
-    }
+    ["-NoProfile", "-NonInteractive", "-Command", script]
   );
   if (result.status !== 0) return null;
   return result.stdout.trim() || null;
 }
 function inspectWindowsProcessTree(rootPid) {
   const script = `$root=${rootPid}; $all=@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId); $ids=New-Object 'System.Collections.Generic.HashSet[int]'; [void]$ids.Add($root); do { $added=$false; foreach($p in $all) { if($ids.Contains([int]$p.ParentProcessId) -and $ids.Add([int]$p.ProcessId)) { $added=$true } } } while($added); $all | Where-Object { $ids.Contains([int]$_.ProcessId) } | Select-Object -ExpandProperty ProcessId`;
-  const result = spawnSync(
+  const result = runBoundedOsCommand(
     "powershell.exe",
-    ["-NoProfile", "-NonInteractive", "-Command", script],
-    {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-      timeout: OS_COMMAND_TIMEOUT_MS,
-      killSignal: "SIGKILL"
-    }
+    ["-NoProfile", "-NonInteractive", "-Command", script]
   );
   if (result.status !== 0) return null;
   return result.stdout.split(/\s+/).filter(Boolean).map(Number).filter((pid) => Number.isInteger(pid) && pid > 0);
@@ -6424,11 +6516,7 @@ function signalRecordedProcessTreeWithResult(pid, signal, expectedExecutable, la
     }
     const args = ["/PID", String(pid), "/T"];
     if (signal === "SIGKILL") args.push("/F");
-    const status = spawnSync("taskkill", args, {
-      stdio: "ignore",
-      timeout: OS_COMMAND_TIMEOUT_MS,
-      killSignal: "SIGKILL"
-    }).status;
+    const status = runBoundedOsCommand("taskkill", args, false).status;
     if (status === 0) return "signalled";
     return isProcessAlive(pid) ? "signal-failed" : absentLeaderResult(pid);
   }
@@ -6599,7 +6687,7 @@ var TaskRunService = class {
       }
       if (current.childPid != null) {
         throw new LegacyRunAbandonConflictError(
-          `run #${runId} \u5DF2\u8BB0\u5F55 child PID ${current.childPid}\uFF0C\u5FC5\u987B\u7531 Worker/Watchdog \u786E\u8BA4\u8FDB\u7A0B\u6811\u9000\u51FA`
+          `run #${runId} \u5DF2\u8BB0\u5F55 child PID ${current.childPid}\uFF0C\u5FC5\u987B\u7531 Worker/Watchdog \u786E\u8BA4\u53D7\u7BA1\u8FDB\u7A0B\u7EC4\u6392\u7A7A`
         );
       }
       if (current.workerPid != null && isProcessAlive(current.workerPid)) {
@@ -6688,7 +6776,7 @@ function runCommandContext(executable, args, cwd) {
 function assertWorkerProcessIsolationSupported(platform = process.platform) {
   if (platform === "win32") {
     throw new Error(
-      "Windows Worker \u5DF2\u5B89\u5168\u7981\u7528\uFF1A\u5F53\u524D\u8FD0\u884C\u65F6\u65E0\u6CD5\u7528 Job Object \u8BC1\u660E\u6574\u4E2A OpenCode \u8FDB\u7A0B\u6811\u5DF2\u9000\u51FA"
+      "Windows Worker \u5DF2\u5B89\u5168\u7981\u7528\uFF1A\u5F53\u524D\u8FD0\u884C\u65F6\u65E0\u6CD5\u7528 Job Object \u63D0\u4F9B\u7B49\u4EF7\u7684\u53D7\u7BA1\u8FDB\u7A0B\u9694\u79BB\u4E0E\u6392\u7A7A\u8BC1\u660E"
     );
   }
 }
@@ -6699,17 +6787,24 @@ var WorkerEngine = class {
   pollTimer = null;
   heartbeatTimer = null;
   pollCyclePromise = null;
+  shutdownDeadlineMs = null;
+  settlementRetryWakeups = /* @__PURE__ */ new Set();
   cfg;
   opencodeBin;
   maxOutputChars;
+  settlementRetryDelaysMs;
+  settlementRetryIntervalMs;
   constructor(cfg, options = {}) {
     this.cfg = cfg.worker;
     this.opencodeBin = options.opencodeBin ?? process.env.SUPERTASK_OPENCODE_BIN ?? "opencode";
     this.maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;
+    this.settlementRetryDelaysMs = options.settlementRetryDelaysMs ?? [250, 1e3, 4e3];
+    this.settlementRetryIntervalMs = options.settlementRetryIntervalMs ?? 5e3;
   }
   start() {
     assertWorkerProcessIsolationSupported();
     this.stopped = false;
+    this.shutdownDeadlineMs = null;
     markGatewayActivity("worker");
     this.poll();
     this.heartbeatTimer = setInterval(() => {
@@ -6718,6 +6813,8 @@ var WorkerEngine = class {
   }
   async stop(gracePeriodMs = 0) {
     this.stopped = true;
+    this.shutdownDeadlineMs = Date.now() + Math.max(0, gracePeriodMs);
+    for (const wake of [...this.settlementRetryWakeups]) wake();
     if (this.pollTimer) {
       clearTimeout(this.pollTimer);
       this.pollTimer = null;
@@ -6787,21 +6884,18 @@ var WorkerEngine = class {
       if (databaseRunningCount >= this.cfg.maxConcurrency) break;
       let task;
       try {
-        task = await TaskService.next({ excludedBatchIds: [...this.activeBatchIds] });
+        task = await TaskService.claimNext({ excludedBatchIds: [...this.activeBatchIds] });
       } catch (err) {
         this.logError("task claim failed", err);
         throw err;
       }
       if (!task) break;
-      if (this.stopped) break;
-      if (!await TaskService.start(task.id)) continue;
-      const batchId = normalizeTaskBatchId(task.batchId);
-      if (batchId) this.activeBatchIds.add(batchId);
       if (this.stopped) {
         await TaskService.resetRunningToPending([task.id]);
-        this.releaseBatch(task);
         break;
       }
+      const batchId = normalizeTaskBatchId(task.batchId);
+      if (batchId) this.activeBatchIds.add(batchId);
       let runId = null;
       try {
         const launchIdentity = `gateway-${process.pid}:launch:${randomUUID()}`;
@@ -7012,7 +7106,7 @@ var WorkerEngine = class {
     if (!entry.guardianDrained) {
       const pid = entry.child.pid;
       const presence = pid == null ? spawnError == null ? "unknown" : "not-running" : inspectSpawnedProcessTreePresence(pid);
-      const message = "guardian \u672A\u63D0\u4F9B\u8FDB\u7A0B\u6811\u6392\u7A7A\u8BC1\u660E";
+      const message = "guardian \u672A\u63D0\u4F9B\u53D7\u7BA1\u8FDB\u7A0B\u7EC4\u6392\u7A7A\u8BC1\u660E";
       if (presence !== "not-running") {
         if (entry.timeoutTimer) {
           clearTimeout(entry.timeoutTimer);
@@ -7048,17 +7142,48 @@ var WorkerEngine = class {
       clearTimeout(entry.timeoutTimer);
       entry.timeoutTimer = null;
     }
-    const settlement = this.commitEntry(entry, code, failure).then(() => true).catch((error) => {
-      markGatewayFailure("worker", error);
-      this.logError("task settlement failed", error, entry.task.id);
-      return false;
-    }).finally(() => {
+    const settlement = this.commitEntryWithRetry(entry, code, failure).finally(() => {
       this.runningTasks.delete(entry.task.id);
       this.releaseBatch(entry.task);
     });
     entry.settlementPromise = settlement;
     return settlement;
   }
+  async commitEntryWithRetry(entry, code, failure) {
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        await this.commitEntry(entry, code, failure);
+        return true;
+      } catch (error) {
+        markGatewayFailure("worker", error);
+        const shortRetryDelayMs = this.settlementRetryDelaysMs[attempt];
+        let retryDelayMs = shortRetryDelayMs ?? this.settlementRetryIntervalMs;
+        if (this.stopped) {
+          const remainingMs = Math.max(0, (this.shutdownDeadlineMs ?? 0) - Date.now());
+          if (remainingMs === 0) return false;
+          retryDelayMs = Math.min(retryDelayMs, remainingMs);
+        }
+        this.logError(
+          `task settlement failed; retrying in ${retryDelayMs}ms`,
+          error,
+          entry.task.id
+        );
+        await this.waitForSettlementRetry(retryDelayMs);
+      }
+    }
+  }
+  waitForSettlementRetry(delayMs) {
+    return new Promise((resolve2) => {
+      let timer;
+      const finish = () => {
+        clearTimeout(timer);
+        this.settlementRetryWakeups.delete(finish);
+        resolve2();
+      };
+      timer = setTimeout(finish, delayMs);
+      this.settlementRetryWakeups.add(finish);
+    });
+  }
   async commitEntry(entry, code, failure) {
     const termination = entry.termination;
     if (termination?.kind === "shutdown") return;