@kognai/orchestrator-core 0.2.4 → 0.2.6

package/dist/lib/sprint-runner-engine.js

@@ -79,6 +79,13 @@ const MAX_HOURS = 6; // kill orchestrator if it runs longer than this
 // Rate limiter: minimum gap between sprint executions (prevents burning Claude 5h limit)
 // Default: 30 min. Override via SPRINT_COOLDOWN_MINUTES env var.
 const COOLDOWN_MINUTES = parseInt(process.env.SPRINT_COOLDOWN_MINUTES ?? '30', 10);
+// TICKET-348 sprint-level backoff: a sprint whose run makes NO forward progress
+// (no pending task reached a terminal/done state) this many times IN A ROW is
+// auto-skipped as 'loop-stuck', so a permanently-failing sprint (e.g. one whose
+// files keep truncating) stops monopolising the runner and the selector advances
+// to other queued work. State lives in .swarm-state/sprint-backoff.json.
+const SPRINT_BACKOFF = (0, path_1.join)(ROOT, '.swarm-state', 'sprint-backoff.json');
+const SPRINT_BACKOFF_THRESHOLD = parseInt(process.env.SPRINT_BACKOFF_THRESHOLD ?? '2', 10);
 // Daily cap: max sprints per calendar day. Default: 100.
 const DAILY_SPRINT_CAP = parseInt(process.env.DAILY_SPRINT_CAP ?? '100', 10);
 // Rolling window cap: max sprints within the last N hours. Default: 20 per 5h.
@@ -354,6 +361,63 @@ function extractSprintNumber(filename) {
 // (Multi-session safety: prevents another Claude session reverting a local file
 // from causing the runner to re-execute paused/done work.)
 const NOTION_OVERRIDE_STATUSES = new Set(['skipped', 'blocked', 'done', 'done-manual', 'loop-stuck', 'rejected']);
+function readBackoff() {
+    try {
+        return JSON.parse((0, fs_1.readFileSync)(SPRINT_BACKOFF, 'utf8'));
+    }
+    catch {
+        return {};
+    }
+}
+function writeBackoff(data) {
+    try {
+        const dir = (0, path_1.join)(ROOT, '.swarm-state');
+        if (!(0, fs_1.existsSync)(dir))
+            (0, fs_1.mkdirSync)(dir, { recursive: true });
+        const tmp = `${SPRINT_BACKOFF}.tmp.${process.pid}`;
+        (0, fs_1.writeFileSync)(tmp, JSON.stringify(data, null, 2));
+        (0, fs_1.renameSync)(tmp, SPRINT_BACKOFF);
+    }
+    catch { /* backoff bookkeeping is best-effort — never block a run */ }
+}
+/** Count pending tasks in a sprint's MERGED view (source + .swarm-state status). */
+function countPendingTasks(sprintPath) {
+    try {
+        return ((0, sprint_state_1.loadSprintMerged)(sprintPath).tasks ?? []).filter((t) => t.status === 'pending').length;
+    }
+    catch {
+        return 0;
+    }
+}
+/** True once a sprint has hit the no-progress threshold — selector skips it. */
+function isBackedOff(sprintId) {
+    const e = readBackoff()[sprintId];
+    return !!e && (e.loop_stuck === true || e.no_progress >= SPRINT_BACKOFF_THRESHOLD);
+}
+/** Post-run: progress = pending count dropped (a task reached terminal/done).
+ *  No progress → increment the consecutive counter; at the threshold, flag
+ *  loop_stuck so findPendingSprint skips it. Any progress → reset. */
+function recordSprintProgress(sprintId, pendingBefore, pendingAfter) {
+    const data = readBackoff();
+    if (pendingAfter < pendingBefore) {
+        if (data[sprintId]) {
+            delete data[sprintId];
+            writeBackoff(data);
+        }
+        return;
+    }
+    const prev = data[sprintId]?.no_progress ?? 0;
+    const next = prev + 1;
+    const loop_stuck = next >= SPRINT_BACKOFF_THRESHOLD;
+    data[sprintId] = { no_progress: next, last: new Date().toISOString(), loop_stuck };
+    writeBackoff(data);
+    if (loop_stuck) {
+        log(`⛔ Backoff: ${sprintId} made no progress ${next}× in a row — marked loop-stuck (auto-skipped). Clear .swarm-state/sprint-backoff.json or fix the sprint to re-enable.`);
+    }
+    else {
+        log(`Backoff: ${sprintId} no progress this run (${next}/${SPRINT_BACKOFF_THRESHOLD} before auto-skip).`);
+    }
+}
 async function findPendingSprint() {
     if (!(0, fs_1.existsSync)(SPRINTS))
         return null;
@@ -404,6 +468,12 @@ async function findPendingSprint() {
                 log(`Skipped ${file}: Notion source-of-truth says '${notionStatus}' (overrides local pending)`);
                 continue;
             }
+            // TICKET-348: local backoff — skip a sprint that has made no forward
+            // progress N runs in a row (loop-stuck), so it can't monopolise the runner.
+            if (isBackedOff(sprintId)) {
+                log(`Skipped ${file}: backoff — no forward progress ${SPRINT_BACKOFF_THRESHOLD}× in a row (loop-stuck, auto-skipped)`);
+                continue;
+            }
             // Dependency check: respect depends_on_sprint — if the upstream sprint
             // has any non-terminal task, skip this one. Founder directive 2026-05-26:
             // swarm must never deadlock on a sprint whose prereqs haven't shipped.
@@ -828,6 +898,10 @@ async function runSprintCycle(opts) {
         return;
     }
     log(`Found pending sprint: ${sprintPath}`);
+    // TICKET-348: snapshot pending count before the run so we can detect whether
+    // this run made any forward progress (and apply backoff if it didn't).
+    const backoffSprintId = (0, path_1.basename)(sprintPath).replace(/\.json$/, '');
+    const pendingBefore = countPendingTasks(sprintPath);
     // TICKET-210: build the ACTIVE sprint from the MERGED view (source definition
     // + .swarm-state status), NOT raw source. The source file holds every task at
     // its authored status (usually 'pending'); reading it directly meant a sprint
@@ -886,15 +960,52 @@ async function runSprintCycle(opts) {
     // overlapping orphans can't accumulate even if cron ever gets re-armed
     const orchestratorTimeoutMs = PER_RUN_HARD_TIMEOUT_MIN * 60 * 1000;
     log(`Spawning orchestrator with ${PER_RUN_HARD_TIMEOUT_MIN}-min hard timeout`);
-    const result = (0, child_process_1.spawnSync)('npx', ['ts-node', orchestratorPath, activePath], {
-        stdio: 'inherit',
-        cwd: ROOT,
-        env: { ...process.env },
-        timeout: orchestratorTimeoutMs,
-        killSignal: 'SIGKILL', // SIGTERM ignored when blocked on subprocess I/O
+    // TICKET-347: spawnSync's `timeout` only SIGKILLs the DIRECT child (`npx`).
+    // Its `ts-node` child + the orchestrator grandchildren (and everything THEY
+    // spawn) get orphaned and keep running — reparented to init. A live incident
+    // (2026-06-13) had a "timed-out" run keep executing for ~2h after the runner
+    // declared it failed, burning ~1.3M tokens + attempting wallet settlements.
+    // Fix: spawn DETACHED so the child leads its own process group, then SIGKILL
+    // the WHOLE group (negative pid) on timeout — taking the entire subtree down.
+    const result = await new Promise((resolveRun) => {
+        const child = (0, child_process_1.spawn)('npx', ['ts-node', orchestratorPath, activePath], {
+            stdio: 'inherit',
+            cwd: ROOT,
+            env: { ...process.env },
+            detached: true, // new process group (setsid) → killable as a unit
+        });
+        let timedOut = false;
+        let settled = false;
+        const finish = (status) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolveRun({ status, timedOut });
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            // Negative pid = signal the entire process group (the detached subtree).
+            try {
+                if (child.pid)
+                    process.kill(-child.pid, 'SIGKILL');
+            }
+            catch { /* group already gone */ }
+            // Belt-and-braces: also target the direct child in case the group call missed.
+            try {
+                child.kill('SIGKILL');
+            }
+            catch { /* already dead */ }
+        }, orchestratorTimeoutMs);
+        child.on('error', (err) => { log(`Orchestrator spawn error: ${err.message}`); finish(1); });
+        child.on('exit', (code) => finish(code));
     });
     const elapsed = Math.round((Date.now() - start) / 60000);
-    const status = result.status === 0 ? '✅ Completed' : `❌ Failed (exit ${result.status})`;
+    const status = result.status === 0
+        ? '✅ Completed'
+        : result.timedOut
+            ? `⏱️ Killed — ${PER_RUN_HARD_TIMEOUT_MIN}-min hard timeout (process group SIGKILLed)`
+            : `❌ Failed (exit ${result.status})`;
     log(`Orchestrator finished: ${status} (${elapsed} min)`);
     // Founder directive 2026-05-26: on non-zero exit, write an incident record
     // and emit an event for CTO + CEO to investigate autonomously. The swarm
@@ -1030,6 +1141,10 @@ async function runSprintCycle(opts) {
     catch {
         // non-fatal
     }
+    // TICKET-348: update sprint backoff. State is fully synced by now (ACTIVE→
+    // .swarm-state + forensic git-log recovery), so countPendingTasks reflects the
+    // post-run truth. No drop in pending = no progress = step toward loop-stuck.
+    recordSprintProgress(backoffSprintId, pendingBefore, countPendingTasks(sprintPath));
     // TICKET-201: post-sprint hook (e.g. dispatch-approved-proposals).
     // Supplied by the product entry so core stays product-agnostic.
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kognai/orchestrator-core",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "Kognai sovereign orchestrator — core engine (template-agnostic). Shared by all products (Kognai/coding, Voxight/market-intel, Invoica/fin-compliance); each supplies only its template. Replaces per-repo forks of orchestrate-agents-v2 / sprint-runner / lib.",
   "license": "MIT",
   "author": "SkinGem",