@kynver-app/runtime 0.1.11 → 0.1.16

package/dist/cli.js

@@ -207,6 +207,18 @@ async function resolveCallbackSecretWithMint(argsSecret, agentOsId, opts) {
     "requires --secret, KYNVER_RUNNER_TOKEN, a scoped runner token (`kynver runner credential`), ~/.kynver/credentials runnerToken, KYNVER_API_KEY with an API base URL to mint one, or (legacy) KYNVER_RUNTIME_SECRET / OPENCLAW_CRON_SECRET"
   );
 }
+async function refreshRunnerToken(agentOsId, opts) {
+  const apiKey = loadApiKey();
+  const baseUrl = resolveConfiguredBaseUrl(opts?.baseUrl);
+  if (!apiKey || !agentOsId || !baseUrl) return null;
+  try {
+    const token = await fetchRunnerCredential(agentOsId, { baseUrl, apiKey });
+    saveRunnerToken(agentOsId, token);
+    return token;
+  } catch {
+    return null;
+  }
+}
 async function fetchRunnerCredential(agentOsId, opts) {
   const apiKey = opts?.apiKey || loadApiKey();
   if (!apiKey) throw new Error("API key required \u2014 run `kynver login` first");
@@ -380,12 +392,12 @@ var DEFAULT_CRITICAL_FREE_BYTES = 15 * 1024 * 1024 * 1024;
 var DEFAULT_MAX_USED_PERCENT = 80;
 var DEFAULT_HARD_MAX_USED_PERCENT = 90;
 function observeRunnerDiskGate(input = {}) {
-  const path14 = input.diskPath?.trim() || "/";
+  const path15 = input.diskPath?.trim() || "/";
   const warnBelowBytes = input.diskFreeWarnBytes ?? DEFAULT_WARN_FREE_BYTES;
   const criticalBelowBytes = input.diskFreeCriticalBytes ?? DEFAULT_CRITICAL_FREE_BYTES;
   const maxUsedPercent = input.diskMaxUsedPercent ?? DEFAULT_MAX_USED_PERCENT;
   const hardMaxUsedPercent = input.diskHardMaxUsedPercent ?? DEFAULT_HARD_MAX_USED_PERCENT;
-  const stats = statfsSync(path14);
+  const stats = statfsSync(path15);
   const freeBytes = Number(stats.bavail) * Number(stats.bsize);
   const totalBytes = Number(stats.blocks) * Number(stats.bsize);
   const usedPercent = totalBytes > 0 ? (totalBytes - freeBytes) / totalBytes * 100 : 100;
@@ -405,7 +417,7 @@ function observeRunnerDiskGate(input = {}) {
   }
   return {
     ok,
-    path: path14,
+    path: path15,
     freeBytes,
     totalBytes,
     usedPercent,
@@ -418,10 +430,12 @@ function observeRunnerDiskGate(input = {}) {
 }
 
 // src/resource-gate.ts
+import { readFileSync as readFileSync5 } from "node:fs";
 import os from "node:os";
 import path5 from "node:path";
 
 // src/run-store.ts
+import { existsSync as existsSync4, readdirSync as readdirSync2 } from "node:fs";
 import path4 from "node:path";
 
 // src/paths.ts
@@ -457,6 +471,20 @@ function loadRun(id) {
   const { runsDir } = getPaths();
   return readJson(path4.join(runDir(runsDir, safeSlug(id)), "run.json"));
 }
+function listRunRecords() {
+  const { runsDir } = getPaths();
+  if (!existsSync4(runsDir)) return [];
+  const runs = [];
+  for (const entry of readdirSync2(runsDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const run = readJson(
+      path4.join(runsDir, entry.name, "run.json"),
+      void 0
+    );
+    if (run?.id) runs.push(run);
+  }
+  return runs;
+}
 function loadWorker(runId, name) {
   const { runsDir } = getPaths();
   return readJson(
@@ -477,7 +505,7 @@ function runDirectory(id) {
 }
 
 // src/heartbeat.ts
-import { existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs";
+import { existsSync as existsSync5, readFileSync as readFileSync3 } from "node:fs";
 function parseHeartbeat(file) {
   const result = {
     heartbeatCount: 0,
@@ -486,7 +514,7 @@ function parseHeartbeat(file) {
     lastHeartbeatSummary: null,
     heartbeatBlocker: null
   };
-  if (!existsSync4(file)) return result;
+  if (!existsSync5(file)) return result;
   const lines = readFileSync3(file, "utf8").split("\n").filter(Boolean);
   for (const line of lines) {
     const entry = safeJson(line);
@@ -502,7 +530,7 @@ function parseHeartbeat(file) {
 }
 
 // src/stream.ts
-import { existsSync as existsSync5, readFileSync as readFileSync4 } from "node:fs";
+import { existsSync as existsSync6, readFileSync as readFileSync4 } from "node:fs";
 function parseClaudeStream(file) {
   const result = {
     firstEventAt: null,
@@ -511,7 +539,7 @@ function parseClaudeStream(file) {
     finalResult: null,
     error: null
   };
-  if (!existsSync5(file)) return result;
+  if (!existsSync6(file)) return result;
   const lines = readFileSync4(file, "utf8").split("\n").filter(Boolean);
   for (const line of lines) {
     const event = safeJson(line);
@@ -804,13 +832,23 @@ function computeAutoMaxWorkers(totalMemBytes, opts = {}) {
   const raw = Math.max(1, Math.floor(budgetBytes / perWorkerMemBytes));
   return Math.min(raw, AUTO_MAX_WORKERS_CEILING);
 }
-function countActiveWorkers(runId) {
-  const run = loadRun(runId);
+function readAvailableMemBytes() {
+  if (process.platform === "linux") {
+    try {
+      const meminfo = readFileSync5("/proc/meminfo", "utf8");
+      const match = meminfo.match(/^MemAvailable:\s+(\d+)\s*kB/m);
+      if (match) return Number(match[1]) * 1024;
+    } catch {
+    }
+  }
+  return os.freemem();
+}
+function countActiveWorkersForRun(run) {
   let active = 0;
   for (const name of Object.keys(run.workers || {})) {
     const worker = readJson(
       path5.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
-      null
+      void 0
     );
     if (!worker) continue;
     const status = computeWorkerStatus(worker);
@@ -820,14 +858,19 @@ function countActiveWorkers(runId) {
   }
   return active;
 }
+function countActiveWorkersGlobal() {
+  let active = 0;
+  for (const run of listRunRecords()) active += countActiveWorkersForRun(run);
+  return active;
+}
 function observeRunnerResourceGate(input) {
   const { perWorkerMemBytes, memReserveBytes, memUtilization, configuredMaxWorkers } = resolveResourceConfig(
     input.config,
     input.configuredMaxWorkersOverride
   );
   const totalMemBytes = input.totalMemBytes ?? os.totalmem();
-  const freeMemBytes = input.freeMemBytes ?? os.freemem();
-  const activeWorkers = input.activeWorkers ?? countActiveWorkers(input.runId);
+  const freeMemBytes = input.freeMemBytes ?? readAvailableMemBytes();
+  const activeWorkers = input.activeWorkers ?? countActiveWorkersGlobal();
   const budgetBytes = Math.max(0, Math.floor(totalMemBytes * memUtilization) - memReserveBytes);
   const capacityFromTotal = Math.max(0, Math.floor(budgetBytes / perWorkerMemBytes));
   const capacityFromFree = Math.max(0, Math.floor(Math.max(0, freeMemBytes - memReserveBytes) / perWorkerMemBytes));
@@ -835,13 +878,13 @@ function observeRunnerResourceGate(input) {
   const targetCap = configuredMaxWorkers ?? autoCap;
   const maxConcurrentWorkers = Math.max(0, Math.min(targetCap, capacityFromTotal));
   const slotsByCapacity = Math.max(0, maxConcurrentWorkers - activeWorkers);
-  const slotsByFreeMem = Math.max(0, capacityFromFree - activeWorkers);
+  const slotsByFreeMem = capacityFromFree;
   const slotsAvailable = Math.min(slotsByCapacity, slotsByFreeMem);
   let reason = null;
   if (slotsAvailable <= 0) {
     if (activeWorkers >= maxConcurrentWorkers) {
       reason = `at worker limit (${activeWorkers}/${maxConcurrentWorkers} running)`;
-    } else if (capacityFromFree <= activeWorkers) {
+    } else if (capacityFromFree <= 0) {
       reason = "insufficient free memory \u2014 waiting for workers to finish";
     } else {
       reason = "no worker slots available";
@@ -864,7 +907,7 @@ function observeRunnerResourceGate(input) {
 }
 
 // src/supervisor.ts
-import { existsSync as existsSync7, mkdirSync as mkdirSync3 } from "node:fs";
+import { existsSync as existsSync8, mkdirSync as mkdirSync3 } from "node:fs";
 import path7 from "node:path";
 
 // src/prompt.ts
@@ -937,19 +980,19 @@ var claudeProvider = {
 };
 
 // src/providers/cursor.ts
-import { closeSync as closeSync2, existsSync as existsSync6, openSync as openSync2, readdirSync as readdirSync2 } from "node:fs";
+import { closeSync as closeSync2, existsSync as existsSync7, openSync as openSync2, readdirSync as readdirSync3 } from "node:fs";
 import { spawn as spawn2 } from "node:child_process";
 import path6 from "node:path";
 var DEFAULT_CURSOR_MODEL = "composer-2.5";
 function latestVersionDir(versionsRoot) {
-  if (!existsSync6(versionsRoot)) return null;
-  const versions = readdirSync2(versionsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory() && /^\d{4}\.\d/.test(entry.name)).map((entry) => entry.name).sort((a, b) => b.localeCompare(a));
+  if (!existsSync7(versionsRoot)) return null;
+  const versions = readdirSync3(versionsRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory() && /^\d{4}\.\d/.test(entry.name)).map((entry) => entry.name).sort((a, b) => b.localeCompare(a));
   return versions[0] ? path6.join(versionsRoot, versions[0]) : null;
 }
 function resolveBundledCursor(versionDir) {
   const nodeExe = path6.join(versionDir, "node.exe");
   const indexJs = path6.join(versionDir, "index.js");
-  if (!existsSync6(nodeExe) || !existsSync6(indexJs)) return null;
+  if (!existsSync7(nodeExe) || !existsSync7(indexJs)) return null;
   return { executable: nodeExe, prefixArgs: [indexJs], shell: false, detached: true };
 }
 function resolveWindowsCursorSpawn(agentBin) {
@@ -972,7 +1015,7 @@ function resolveAgentBin() {
   if (configured) return configured;
   if (process.platform === "win32") {
     const localAgent = path6.join(process.env.LOCALAPPDATA || "", "cursor-agent", "agent.cmd");
-    if (existsSync6(localAgent)) return localAgent;
+    if (existsSync7(localAgent)) return localAgent;
   }
   return "agent";
 }
@@ -1041,8 +1084,11 @@ function resolveWorkerProvider(name) {
 
 // src/supervisor.ts
 function spawnWorkerProcess(run, opts) {
-  const name = safeSlug(opts.name);
-  if (!opts.name) throw new Error("worker name is required");
+  const rawName = typeof opts.name === "string" ? opts.name.trim() : "";
+  if (!rawName || rawName === "undefined" || rawName === "null") {
+    throw new Error(`worker name is required and must be a real identifier (got: ${JSON.stringify(opts.name)})`);
+  }
+  const name = safeSlug(rawName);
   if (run.workers?.[name]) throw new Error(`worker already exists in run ${run.id}: ${name}`);
   if (!opts.task) throw new Error(`missing task text for worker ${name}`);
   const { worktreesDir } = getPaths();
@@ -1050,7 +1096,7 @@ function spawnWorkerProcess(run, opts) {
   mkdirSync3(workerDir, { recursive: true });
   const worktreePath = path7.join(worktreesDir, run.id, name);
   const branch = opts.branch || `agent/${run.id}/${name}`;
-  if (existsSync7(worktreePath)) throw new Error(`worktree path already exists: ${worktreePath}`);
+  if (existsSync8(worktreePath)) throw new Error(`worktree path already exists: ${worktreePath}`);
   git(run.repo, ["fetch", "origin", "--prune"], { allowFailure: true });
   git(run.repo, ["worktree", "add", "-b", branch, worktreePath, run.baseCommit], { throwError: true });
   const stdoutPath = path7.join(workerDir, "stdout.jsonl");
@@ -1112,6 +1158,11 @@ function spawnWorkerProcess(run, opts) {
 }
 function startWorker(args) {
   const run = loadRun(String(args.run));
+  const name = typeof args.name === "string" ? args.name.trim() : "";
+  if (!name) {
+    console.error("worker start failed: --name is required");
+    process.exit(1);
+  }
   const task = args.task ? String(args.task) : readMaybeFile(args.taskFile ? String(args.taskFile) : void 0);
   if (!task) {
     console.error("missing --task or --task-file");
@@ -1119,7 +1170,7 @@ function startWorker(args) {
   }
   try {
     const worker = spawnWorkerProcess(run, {
-      name: String(args.name),
+      name,
       task,
       ownedPaths: args.owned ? String(args.owned).split(",").map((s) => s.trim()).filter(Boolean) : [],
       model: args.model ? String(args.model) : void 0,
@@ -1316,7 +1367,7 @@ async function sweepRun(args) {
     for (const name of Object.keys(run.workers || {})) {
       const worker = readJson(
         path8.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
-        null
+        void 0
       );
       if (!worker || !worker.dispatched || !worker.taskId) continue;
       const status = computeWorkerStatus(worker);
@@ -1358,7 +1409,7 @@ async function sweepRun(args) {
 }
 
 // src/worktree.ts
-import { existsSync as existsSync8, mkdirSync as mkdirSync4 } from "node:fs";
+import { existsSync as existsSync9, mkdirSync as mkdirSync4 } from "node:fs";
 import path10 from "node:path";
 
 // src/validate.ts
@@ -1381,7 +1432,7 @@ function createRun(args) {
   ensureGitRepo(repo);
   const id = args.id ? validateRunId(String(args.id)) : timestampSlug(String(args.name || "run"));
   const dir = runDirectory(id);
-  if (existsSync8(dir)) failExists(`run already exists: ${id}`);
+  if (existsSync9(dir)) failExists(`run already exists: ${id}`);
   mkdirSync4(dir, { recursive: true });
   const base = String(args.base || "origin/main");
   const baseCommit = git(repo, ["rev-parse", base]).trim();
@@ -1400,7 +1451,7 @@ function createRun(args) {
 }
 function listRuns() {
   const { runsDir } = getPaths();
-  const rows = listRunIds(runsDir).map((id) => readJson(path10.join(runDirectory(id), "run.json"), null)).filter(Boolean).map((run) => ({
+  const rows = listRunIds(runsDir).map((id) => readJson(path10.join(runDirectory(id), "run.json"), void 0)).filter(Boolean).map((run) => ({
     id: run.id,
     name: run.name,
     status: run.status,
@@ -1416,6 +1467,34 @@ function failExists(message) {
 
 // src/worker-ops.ts
 import path11 from "node:path";
+async function postCompletion(url, secret, body) {
+  const res = await fetch(url, {
+    method: "POST",
+    headers: buildHarnessCallbackHeaders(secret),
+    body: JSON.stringify(body)
+  });
+  let parsed = null;
+  try {
+    parsed = await res.json();
+  } catch {
+    parsed = null;
+  }
+  return { ok: res.ok, status: res.status, parsed };
+}
+function completionErrorText(parsed) {
+  if (parsed && typeof parsed === "object") {
+    const err = parsed.error;
+    if (typeof err === "string" && err.trim()) return err.trim();
+  }
+  return void 0;
+}
+function persistCompletionBlocker(worker, reason) {
+  const current = worker.completionBlocker;
+  if ((current ?? void 0) === (reason ?? void 0)) return;
+  if (reason) worker.completionBlocker = reason;
+  else delete worker.completionBlocker;
+  saveWorker(worker.runId, worker);
+}
 async function tryCompleteWorker(args) {
   const worker = loadWorker(String(args.run), String(args.name));
   const status = computeWorkerStatus(worker);
@@ -1428,7 +1507,8 @@ async function tryCompleteWorker(args) {
     return { ok: true, skipped: true, reason: "worker-not-finished" };
   }
   const base = resolveBaseUrl(args.baseUrl ? String(args.baseUrl) : void 0);
-  const secret = await resolveCallbackSecretWithMint(args.secret ? String(args.secret) : void 0, agentOsId, { baseUrl: base });
+  const explicitSecret = args.secret ? String(args.secret) : void 0;
+  let secret = await resolveCallbackSecretWithMint(explicitSecret, agentOsId, { baseUrl: base });
   const url = `${base}/api/agent-os/by-id/${encodeURIComponent(agentOsId)}/harness/completion`;
   const body = {
     source: "openclaw-harness",
@@ -1440,18 +1520,23 @@ async function tryCompleteWorker(args) {
     finishedAt: status.lastActivityAt || (/* @__PURE__ */ new Date()).toISOString(),
     status
   };
-  const res = await fetch(url, {
-    method: "POST",
-    headers: buildHarnessCallbackHeaders(secret),
-    body: JSON.stringify(body)
-  });
-  let parsed = null;
-  try {
-    parsed = await res.json();
-  } catch {
-    parsed = null;
+  let result = await postCompletion(url, secret, body);
+  if ((result.status === 401 || result.status === 403) && !explicitSecret) {
+    const refreshed = await refreshRunnerToken(agentOsId, { baseUrl: base });
+    if (refreshed && refreshed !== secret) {
+      secret = refreshed;
+      result = await postCompletion(url, secret, body);
+    }
+  }
+  if (result.ok) {
+    persistCompletionBlocker(worker, void 0);
+    return { ok: true, httpStatus: result.status, response: result.parsed };
   }
-  return { ok: res.ok, httpStatus: res.status, response: parsed };
+  const authRejected = result.status === 401 || result.status === 403;
+  const detail = completionErrorText(result.parsed) ?? (authRejected ? "runner token unauthorized" : "non-2xx response");
+  const reason = authRejected ? `completion replay rejected (${result.status}): ${detail}` : `completion replay failed (${result.status}): ${detail}`;
+  persistCompletionBlocker(worker, reason);
+  return { ok: false, httpStatus: result.status, response: result.parsed, completionBlocked: true };
 }
 async function completeWorker(args) {
   try {
@@ -1513,17 +1598,19 @@ function runStatus(args) {
   const workers = names.map((name) => {
     const worker = readJson(
       path11.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
-      null
+      void 0
     );
     if (!worker) {
       return { worker: name, status: "missing", attention: "needs_attention", attentionReason: "worker.json not found" };
     }
     const status = computeWorkerStatus(worker, { base: run.base });
+    const rawBlocker = worker.completionBlocker;
+    const completionBlocker = typeof rawBlocker === "string" && rawBlocker ? rawBlocker : void 0;
     return {
       worker: status.worker,
-      status: status.status,
-      attention: status.attention.state,
-      attentionReason: status.attention.reason,
+      status: completionBlocker ? "blocked" : status.status,
+      attention: completionBlocker ? "blocked" : status.attention.state,
+      attentionReason: completionBlocker ?? status.attention.reason,
       pid: status.pid,
       alive: status.alive,
       currentTool: status.currentTool,
@@ -1579,10 +1666,53 @@ function stopWorker(args) {
 }
 
 // src/pipeline-tick.ts
-import path13 from "node:path";
+import path14 from "node:path";
 
-// src/plan-progress-daemon-sync.ts
+// src/finalize.ts
 import path12 from "node:path";
+var ACTIVE_RUN_STATUSES = /* @__PURE__ */ new Set(["running", "dispatching", "pending", "queued"]);
+function terminalStatusFor(run) {
+  const names = Object.keys(run.workers || {});
+  if (names.length === 0) return "failed";
+  let anyAlive = false;
+  let anyResult = false;
+  let anyCompletionBlocked = false;
+  for (const name of names) {
+    const worker = readJson(
+      path12.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
+      void 0
+    );
+    if (!worker) continue;
+    const status = computeWorkerStatus(worker);
+    if (status.alive && !status.finalResult) {
+      anyAlive = true;
+      break;
+    }
+    if (typeof worker.completionBlocker === "string" && worker.completionBlocker) {
+      anyCompletionBlocked = true;
+    }
+    if (status.finalResult) anyResult = true;
+  }
+  if (anyAlive) return null;
+  if (anyCompletionBlocked) return null;
+  return anyResult ? "completed" : "failed";
+}
+function finalizeStaleRuns() {
+  const finalized = [];
+  for (const run of listRunRecords()) {
+    if (!ACTIVE_RUN_STATUSES.has(run.status)) continue;
+    const next = terminalStatusFor(run);
+    if (!next || next === run.status) continue;
+    const from = run.status;
+    run.status = next;
+    saveRun(run);
+    finalized.push({ runId: run.id, from, to: next });
+  }
+  return finalized;
+}
+
+// src/plan-progress-daemon-sync.ts
+import path13 from "node:path";
 
 // src/plan-progress-sync.ts
 async function syncPlanProgress(args) {
@@ -1606,8 +1736,8 @@ async function syncActiveWorkerPlanProgress(runId, args) {
   const outcomes = [];
   for (const name of Object.keys(run.workers || {})) {
     const worker = readJson(
-      path12.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
-      null
+      path13.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
+      void 0
     );
     if (!worker?.dispatched || !worker.taskId) continue;
     const status = computeWorkerStatus(worker);
@@ -1660,13 +1790,13 @@ async function completeFinishedWorkers(runId, args) {
   const outcomes = [];
   for (const name of Object.keys(run.workers || {})) {
     const worker = readJson(
-      path13.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
-      null
+      path14.join(runDirectory(run.id), "workers", safeSlug(name), "worker.json"),
+      void 0
     );
-    if (!worker?.dispatched || !worker.taskId) continue;
+    if (!worker?.taskId) continue;
     const status = computeWorkerStatus(worker);
     if (!isFinishedWorkerStatus(status)) continue;
-    if (!status.finalResult) continue;
+    if (!worker.dispatched && !status.finalResult) continue;
     const result = await tryCompleteWorker({
       run: runId,
       name,
@@ -1695,6 +1825,7 @@ async function runPipelineTick(args) {
   const execute = args.execute !== false && args.execute !== "false";
   runStatus({ run: runId });
   const completedWorkers = await completeFinishedWorkers(runId, args);
+  const finalizedStaleRuns = finalizeStaleRuns();
   const planProgressSync = await syncActiveWorkerPlanProgress(runId, args);
   const workspacePrefs = await fetchWorkspaceRuntimePreferences(agentOsId, args);
   const resourceGate = observeRunnerResourceGate({
@@ -1735,6 +1866,7 @@ async function runPipelineTick(args) {
     execute,
     resourceGate,
     completedWorkers,
+    finalizedStaleRuns,
     planProgressSync,
     operatorTick,
     sweep,