@aion0/forge 0.9.0 → 0.9.2

package/lib/jobs/scheduler.ts

@@ -9,27 +9,139 @@
 
 import {
   ensureSchema, getDueJobs, hasInflightRun, startRun, finishRun,
-  markSeen, isSeen, recordDispatch, getJob, updateJob,
+  markSeen, isSeen, recordDispatch, getJob, updateJob, setNextRunAt,
 } from './store';
+import { CronExpressionParser } from 'cron-parser';
+import { ensureInstalledInProject } from '../skills';
 import type { Job, JobRunStatus, PipelineDispatchParams, ChatDispatchParams } from './types';
 import { dispatchTool } from '@/lib/chat/tool-dispatcher';
 import { dispatchToPipeline, dispatchToChat, dispatchToChatSummary } from './dispatcher';
 import { getDb } from '@/src/core/db/database';
 import { getDbPath } from '@/src/config';
+import { existsSync, readFileSync } from 'node:fs';
+import { join as joinPath } from 'node:path';
+import { getDataDir } from '@/lib/dirs';
+
+/** Reconcile stale pipeline_runs rows against the canonical JSON
+ *  state on disk. Called before any count, so counts return real
+ *  inflight numbers — not zombies left behind by missed
+ *  syncRunStatus calls (process crash / ReferenceError swallowed by
+ *  empty catch / etc).
+ *
+ *  Rules:
+ *    - Rows older than 30s in 'running'/'pending' get checked
+ *      (younger ones might genuinely not have written first state).
+ *    - JSON missing → mark DB row 'failed' (pipeline was cleaned up).
+ *    - JSON in terminal state → sync DB row to that state.
+ *    - JSON still running → leave the DB row.
+ *
+ *  Costs one stat + small file read per stale row. Idempotent.
+ */
+function reconcileStalePipelineRuns(): void {
+  try {
+    const db = getDb(getDbPath());
+    const stale = db.prepare(
+      `SELECT id, pipeline_id FROM pipeline_runs
+        WHERE status IN ('running', 'pending')
+          AND datetime(created_at) < datetime('now', '-30 seconds')`,
+    ).all() as { id: string; pipeline_id: string }[];
+
+    if (stale.length === 0) return;
+
+    const update = db.prepare(`UPDATE pipeline_runs SET status = ? WHERE id = ?`);
+    const pipelineDir = joinPath(getDataDir(), 'pipelines');
+
+    for (const row of stale) {
+      const file = joinPath(pipelineDir, `${row.pipeline_id}.json`);
+      if (!existsSync(file)) {
+        update.run('failed', row.id);
+        console.warn(`[scheduler] reconciled zombie pipeline_run ${row.id} → failed (json gone)`);
+        continue;
+      }
+      try {
+        const p = JSON.parse(readFileSync(file, 'utf8')) as { status?: string };
+        if (p.status && p.status !== 'running' && p.status !== 'pending') {
+          update.run(p.status, row.id);
+          console.warn(`[scheduler] reconciled pipeline_run ${row.id} → ${p.status}`);
+        }
+      } catch (e) {
+        console.warn(`[scheduler] failed to read ${file} during reconcile: ${(e as Error).message}`);
+      }
+    }
+  } catch (e) {
+    console.warn(`[scheduler] reconcileStalePipelineRuns failed: ${(e as Error).message}`);
+  }
+}
 
-/** Count pipelines currently running or pending. Used as the global
- *  concurrency budget — paired with settings.maxConcurrentPipelines. */
+/** Count pipelines currently running or pending — global. Used to
+ *  enforce maxConcurrentPipelines. Reconciles stale rows first. */
 function countActivePipelines(): number {
+  reconcileStalePipelineRuns();
   try {
     const r = getDb(getDbPath()).prepare(
       `SELECT COUNT(*) AS n FROM pipeline_runs WHERE status IN ('running', 'pending')`,
     ).get() as { n: number } | undefined;
     return r?.n ?? 0;
-  } catch {
+  } catch (e) {
+    console.warn(`[scheduler] countActivePipelines failed: ${(e as Error).message}`);
+    return 0;
+  }
+}
+
+/** Count THIS Job's previously-dispatched pipelines that are still
+ *  running or pending. Used by sequential mode to gate the next tick.
+ *  Reconciles stale rows first — without that, zombie 'running'
+ *  rows from previous Forge crashes would block sequential Jobs forever. */
+function countMyInflightPipelines(jobId: string): number {
+  reconcileStalePipelineRuns();
+  try {
+    const r = getDb(getDbPath()).prepare(`
+      SELECT COUNT(*) AS n
+        FROM pipeline_runs pr
+       WHERE pr.status IN ('running', 'pending')
+         AND pr.pipeline_id IN (
+           SELECT jd.dispatch_target_id
+             FROM job_dispatches jd
+             JOIN job_runs        jr ON jr.id = jd.job_run_id
+            WHERE jr.job_id = ?
+              AND jd.dispatch_type = 'pipeline'
+              AND jd.created_at > datetime('now', '-1 day')
+         )
+    `).get(jobId) as { n: number } | undefined;
+    return r?.n ?? 0;
+  } catch (e) {
+    console.warn(`[scheduler] countMyInflightPipelines(${jobId}) failed: ${(e as Error).message}`);
     return 0;
   }
 }
 
+/** "Is this Job busy right now?" — used by:
+ *   1. Manual fire endpoint to refuse double-clicks (return 409).
+ *   2. GET /api/jobs to render disabled state on Run / Force buttons.
+ *
+ *  Busy ⇔
+ *    - there's an inflight job_run (tick currently executing) OR
+ *    - it's a sequential Job whose previously-dispatched pipeline
+ *      is still running/pending.
+ *
+ *  Reconciles stale rows before checking so zombies don't pin a
+ *  Job as "busy" forever.
+ */
+export function isJobBusy(jobId: string): { busy: boolean; reason: string } {
+  if (hasInflightRun(jobId)) {
+    return { busy: true, reason: 'a tick of this Job is currently executing' };
+  }
+  const job = getJob(jobId);
+  // Default-or-explicit sequential — check pipeline inflight.
+  if (job && (job as any).concurrency_mode !== 'parallel' && job.dispatch_type === 'pipeline') {
+    const n = countMyInflightPipelines(jobId);
+    if (n > 0) {
+      return { busy: true, reason: `${n} pipeline${n === 1 ? '' : 's'} from a prior run still active (sequential mode)` };
+    }
+  }
+  return { busy: false, reason: '' };
+}
+
 /** Read settings.maxConcurrentPipelines (default 5, ceiling 20). */
 async function getMaxConcurrentPipelines(): Promise<number> {
   try {
@@ -72,7 +184,7 @@ async function tick(): Promise<void> {
     // Kick off the run; don't await — long connector calls / pipeline triggers
     // shouldn't block the scheduler loop.
     const { runId } = prepareRun(job, 'schedule');
-    void executeRun(job, runId).catch((e) => {
+    void executeRun(job, runId, 'schedule').catch((e) => {
       console.error(`[jobs] runJob ${job.id} crashed`, e);
     });
   }
@@ -88,7 +200,6 @@ function toSqlIso(d: Date): string {
  * don't fire repeatedly when their schedule_at time is in the past.
  */
 function advanceSchedule(job: Job): void {
-  const { setNextRunAt } = require('./store') as typeof import('./store');
   const now = Date.now();
 
   if (job.schedule_kind === 'manual') {
@@ -109,7 +220,6 @@ function advanceSchedule(job: Job): void {
 
   if (job.schedule_kind === 'cron' && job.schedule_cron) {
     try {
-      const { CronExpressionParser } = require('cron-parser');
       const iter = CronExpressionParser.parse(job.schedule_cron, { currentDate: new Date(now) });
       const next = iter.next().toDate();
       setNextRunAt(job.id, toSqlIso(next));
@@ -134,7 +244,7 @@ function advanceSchedule(job: Job): void {
  */
 export async function runJob(jobOrId: Job | string, trigger: 'schedule' | 'manual'): Promise<string> {
   const { job, runId } = prepareRun(jobOrId, trigger);
-  await executeRun(job, runId);
+  await executeRun(job, runId, trigger);
   return runId;
 }
 
@@ -158,7 +268,7 @@ export function prepareRun(jobOrId: Job | string, trigger: 'schedule' | 'manual'
  * we also mirror the high-level lines to console for live tailing via
  * `tail -f forge.log | grep [jobs]`.
  */
-export async function executeRun(job: Job, runId: string): Promise<void> {
+export async function executeRun(job: Job, runId: string, trigger: 'schedule' | 'manual' = 'schedule'): Promise<void> {
   const t0 = Date.now();
   let itemsSeen = 0, itemsNew = 0, itemsDispatched = 0;
   let runError: string | null = null;
@@ -341,12 +451,74 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
     //       monopolizing all slots.
     // Why both: a single job with max_per_tick=10 can still go over if
     // there are already 15 pipelines from OTHER jobs in flight.
-    const budget = (() => {
+    const concurrencyMode: 'parallel' | 'sequential' =
+      (job as any).concurrency_mode === 'parallel' ? 'parallel' : 'sequential';
+    const budget = concurrencyMode === 'sequential' ? 1 : (() => {
       const v = (job as any).max_per_tick;
       if (!Number.isFinite(v) || v == null) return 5;
       return Math.min(Math.max(Math.trunc(v), 1), 10);
     })();
     const globalCap = await getMaxConcurrentPipelines();
+
+    // Sequential gate: if any pipeline this Job previously dispatched
+    // is still running, defer the entire tick. Items stay un-dedup-
+    // marked so the next tick re-encounters them. This guarantees at
+    // most one pipeline from this Job runs at a time — solves
+    // GitLab/Mantis rate-limit + browser-tab race classes of bugs.
+    if (concurrencyMode === 'sequential' && job.dispatch_type === 'pipeline') {
+      // on_failure='stop': if the most recent dispatched pipeline ended
+      // in 'failed', halt the drain. User has to Force-run to resume.
+      // Default 'continue' just falls through to the regular gate check.
+      //
+      // Manual fires (Run now / Force run) bypass this check — they
+      // ARE the user's "resume after a failure" action; halting them
+      // would deadlock the Job.
+      const onFailure: 'continue' | 'stop' = (job as any).on_failure === 'stop' ? 'stop' : 'continue';
+      if (onFailure === 'stop' && trigger !== 'manual') {
+        try {
+          const recent = getDb(getDbPath()).prepare(`
+            SELECT pr.status FROM pipeline_runs pr
+              JOIN job_dispatches jd ON jd.dispatch_target_id = pr.pipeline_id
+              JOIN job_runs       jr ON jr.id = jd.job_run_id
+             WHERE jr.job_id = ?
+               AND jd.dispatch_type = 'pipeline'
+             ORDER BY jd.created_at DESC LIMIT 1
+          `).get(job.id) as { status?: string } | undefined;
+          if (recent?.status === 'failed') {
+            logLine('warn', `on_failure=stop: previous pipeline FAILED — halting sequential drain. Clear with Force run.`);
+            try { setNextRunAt(job.id, null); } catch {}
+            persist({
+              status: 'ok',
+              notes: `Halted: on_failure=stop and previous pipeline failed. Force run to resume.`,
+            });
+            return;
+          }
+        } catch (e) {
+          logLine('warn', `on_failure check failed: ${(e as Error).message} — proceeding as if 'continue'`);
+        }
+      }
+
+      const myInflight = countMyInflightPipelines(job.id);
+      if (myInflight > 0) {
+        logLine('info', `sequential mode: ${myInflight} pipeline from this Job still running — deferring entire tick`);
+        // Drain-mode: schedule another tick in 60s so the queue keeps
+        // draining regardless of schedule_kind. Without this a manual
+        // Job whose advanceSchedule cleared next_run_at would never get
+        // re-picked-up, and the deferred items would sit forever.
+        try {
+          const nextDrain = new Date(Date.now() + 60_000);
+          setNextRunAt(job.id, toSqlIso(nextDrain));
+        } catch (e) {
+          logLine('warn', `sequential drain (gate): failed to set next_run_at: ${(e as Error).message}`);
+        }
+        persist({
+          status: 'ok',
+          notes: `Sequential: previous pipeline still inflight (${myInflight}). All ${itemsArr.length} item(s) deferred — will retry next tick.`,
+        });
+        return;
+      }
+    }
+
     let dispatchedThisTick = 0;
     let dedupHits = 0, missingKey = 0, deferred = 0;
     for (const [idx, item] of itemsArr.entries()) {
@@ -393,7 +565,6 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
         if (targetProject) {
           for (const skillName of job.skills) {
             try {
-              const { ensureInstalledInProject } = require('../skills');
               const r = await ensureInstalledInProject(skillName, targetProject);
               if (!r.installed) logLine('warn', `skill "${skillName}" not installable: ${r.reason}`);
             } catch (err) {
@@ -437,6 +608,24 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
       const baseNote = note ? note + ' ' : '';
       note = `${baseNote}${deferred} item(s) deferred to next tick (per-Job budget ${budget} or global cap ${globalCap} reached).`;
     }
+
+    // Sequential drain mode: if this Job is sequential AND has deferred
+    // items waiting, force next_run_at to a short interval so the scheduler
+    // keeps picking it up until the batch is drained. Works for ANY
+    // schedule_kind — including 'manual' Jobs where the user did one
+    // Force run and expects all batched items to process automatically
+    // afterwards. Without this, manual + sequential = "one Force run
+    // dispatches exactly one item, stop" — which surprised the user.
+    if (deferred > 0 && concurrencyMode === 'sequential' && job.dispatch_type === 'pipeline') {
+      try {
+        const nextDrain = new Date(Date.now() + 60_000); // 60s — tick cycle
+        setNextRunAt(job.id, toSqlIso(nextDrain));
+        logLine('info', `sequential drain: ${deferred} item(s) still queued — next tick at ${nextDrain.toISOString()}`);
+      } catch (e) {
+        logLine('warn', `sequential drain: failed to set next_run_at: ${(e as Error).message}`);
+      }
+    }
+
     logLine('info', `tick done in ${Date.now() - t0}ms — ${itemsSeen} seen, ${itemsNew} new, ${itemsDispatched} dispatched, ${dedupHits} dedup hits` + (deferred ? `, ${deferred} deferred` : '') + (missingKey ? `, ${missingKey} missing-key` : ''));
     persist({ status: 'ok', notes: note });
   } catch (e) {
@@ -474,6 +663,21 @@ function pickPath(obj: unknown, path: string): unknown {
 }
 
 function pickDedupKey(item: unknown, field: string): string | null {
+  // `field` can be a single dot-path ("user.id") OR a colon-joined
+  // composite of multiple dot-paths ("iid:user_notes_count"). The
+  // composite form yields a stable signature for "this entity's
+  // current change state" — e.g. an MR id + its comment count, so a
+  // new comment bumps the signature and triggers a fresh dispatch.
+  // Any segment missing → null (caller skips item).
+  if (field.includes(':')) {
+    const parts: string[] = [];
+    for (const seg of field.split(':')) {
+      const v = pickPath(item, seg);
+      if (v == null) return null;
+      parts.push(typeof v === 'string' ? v : String(v));
+    }
+    return parts.join(':');
+  }
   const v = pickPath(item, field);
   if (v == null) return null;
   return typeof v === 'string' ? v : String(v);

package/lib/jobs/store.ts

@@ -7,6 +7,7 @@ import { getDb } from '@/src/core/db/database';
 import { getDbPath } from '@/src/config';
 import { randomUUID } from 'node:crypto';
 import { toIsoUTC } from '@/lib/iso-time';
+import { CronExpressionParser } from 'cron-parser';
 import type {
   Job, JobRun, JobDispatch, CreateJobInput,
   JobRunStatus, JobRunTrigger, JobDispatchStatus,
@@ -46,6 +47,14 @@ export function ensureSchema(): void {
           to protect against catastrophic fan-out (e.g. mantis search
           returning 200 bugs and spawning 200 worktrees). */
       max_per_tick                INTEGER NOT NULL DEFAULT 5,
+      /** 'sequential' (default) | 'sequential' — see Job.concurrency_mode
+          in types.ts. Sequential mode dispatches one pipeline at a
+          time and waits for it to reach a terminal state before
+          starting the next. */
+      concurrency_mode            TEXT NOT NULL DEFAULT 'sequential',
+      /** 'continue' (default) | 'stop' — what to do when a dispatched
+          pipeline fails in sequential drain. See Job.on_failure. */
+      on_failure                  TEXT NOT NULL DEFAULT 'continue',
       last_run_at                 TEXT,
       next_run_at                 TEXT,
       created_at                  TEXT NOT NULL DEFAULT (datetime('now')),
@@ -95,6 +104,8 @@ export function ensureSchema(): void {
   try { db().exec(`ALTER TABLE jobs ADD COLUMN schedule_at TEXT`); } catch {}
   try { db().exec(`ALTER TABLE jobs ADD COLUMN schedule_cron TEXT`); } catch {}
   try { db().exec(`ALTER TABLE jobs ADD COLUMN max_per_tick INTEGER NOT NULL DEFAULT 5`); } catch {}
+  try { db().exec(`ALTER TABLE jobs ADD COLUMN concurrency_mode TEXT NOT NULL DEFAULT 'sequential'`); } catch {}
+  try { db().exec(`ALTER TABLE jobs ADD COLUMN on_failure TEXT NOT NULL DEFAULT 'continue'`); } catch {}
   ensured = true;
 }
 
@@ -118,6 +129,8 @@ function rowToJob(r: any): Job {
     schedule_at: toIsoUTC(r.schedule_at),
     schedule_cron: r.schedule_cron || null,
     max_per_tick: typeof r.max_per_tick === 'number' ? r.max_per_tick : 5,
+    concurrency_mode: r.concurrency_mode === 'parallel' ? 'parallel' : 'sequential',
+    on_failure: r.on_failure === 'stop' ? 'stop' : 'continue',
     last_run_at: toIsoUTC(r.last_run_at),
     next_run_at: toIsoUTC(r.next_run_at),
     created_at: toIsoUTC(r.created_at) || r.created_at,
@@ -193,8 +206,9 @@ export function createJob(input: CreateJobInput): Job {
       source_connector, source_tool, source_input,
       items_path, dedup_field,
       dispatch_type, dispatch_params, skills,
-      schedule_kind, schedule_at, schedule_cron, max_per_tick)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      schedule_kind, schedule_at, schedule_cron, max_per_tick,
+      concurrency_mode, on_failure)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `).run(
     id,
     input.name,
@@ -212,6 +226,8 @@ export function createJob(input: CreateJobInput): Job {
     input.schedule_at || null,
     input.schedule_cron || null,
     clampMaxPerTick(input.max_per_tick),
+    input.concurrency_mode === 'parallel' ? 'parallel' : 'sequential',
+    input.on_failure === 'stop' ? 'stop' : 'continue',
   );
 
   // Backfill guard: if mark_existing_as_seen is true (default), we don't pre-seed
@@ -232,7 +248,6 @@ export function createJob(input: CreateJobInput): Job {
     if (!Number.isNaN(t.getTime())) setNextRunAt(id, t.toISOString().replace('T', ' ').slice(0, 19));
   } else if (input.schedule_kind === 'cron' && input.schedule_cron) {
     try {
-      const { CronExpressionParser } = require('cron-parser');
       const iter = CronExpressionParser.parse(input.schedule_cron, { currentDate: new Date() });
       const next = iter.next().toDate();
       setNextRunAt(id, next.toISOString().replace('T', ' ').slice(0, 19));
@@ -252,6 +267,8 @@ export function updateJob(id: string, patch: Partial<{
   schedule_at: string | null;
   schedule_cron: string | null;
   max_per_tick: number;
+  concurrency_mode: 'parallel' | 'sequential';
+  on_failure: 'continue' | 'stop';
 }>): boolean {
   ensureSchema();
   const sets: string[] = []; const vals: any[] = [];
@@ -270,6 +287,14 @@ export function updateJob(id: string, patch: Partial<{
   if (patch.schedule_at !== undefined) { sets.push('schedule_at = ?'); vals.push(patch.schedule_at); }
   if (patch.schedule_cron !== undefined) { sets.push('schedule_cron = ?'); vals.push(patch.schedule_cron); }
   if (patch.max_per_tick !== undefined) { sets.push('max_per_tick = ?'); vals.push(clampMaxPerTick(patch.max_per_tick)); }
+  if (patch.concurrency_mode !== undefined) {
+    sets.push('concurrency_mode = ?');
+    vals.push(patch.concurrency_mode === 'parallel' ? 'parallel' : 'sequential');
+  }
+  if (patch.on_failure !== undefined) {
+    sets.push('on_failure = ?');
+    vals.push(patch.on_failure === 'stop' ? 'stop' : 'continue');
+  }
   if (sets.length === 0) return false;
   sets.push("updated_at = datetime('now')");
   vals.push(id);
@@ -396,3 +421,54 @@ export function listDispatches(runId: string): JobDispatch[] {
   const rows = db().prepare('SELECT * FROM job_dispatches WHERE job_run_id = ? ORDER BY created_at ASC').all(runId) as any[];
   return rows.map(rowToDispatch);
 }
+
+/** Recent pipelines this Job has dispatched, decorated with live
+ *  pipeline_runs status. Used by the Job row to show what's running,
+ *  what's done, what failed — without making the user navigate to the
+ *  Pipeline view for each. Capped at N per call to keep the UI fast. */
+export interface JobDispatchedPipeline {
+  dispatch_id: string;
+  job_run_id: string;
+  item_key: string;
+  item_preview: string | null;
+  pipeline_id: string;
+  pipeline_status: string;  // 'running' | 'pending' | 'done' | 'failed' | 'cancelled' | 'unknown'
+  workflow_name: string | null;
+  dispatched_at: string;
+}
+
+export function listJobDispatchedPipelines(jobId: string, limit = 20): JobDispatchedPipeline[] {
+  ensureSchema();
+  const rows = db().prepare(`
+    SELECT jd.id AS dispatch_id, jd.job_run_id, jd.item_key, jd.item_preview,
+           jd.dispatch_target_id AS pipeline_id, jd.created_at AS dispatched_at,
+           pr.status AS pipeline_status, pr.workflow_name AS workflow_name
+      FROM job_dispatches jd
+      JOIN job_runs        jr ON jr.id = jd.job_run_id
+ LEFT JOIN pipeline_runs   pr ON pr.pipeline_id = jd.dispatch_target_id
+     WHERE jr.job_id = ?
+       AND jd.dispatch_type = 'pipeline'
+     ORDER BY jd.created_at DESC
+     LIMIT ?
+  `).all(jobId, limit) as any[];
+  return rows.map((r) => ({
+    dispatch_id: r.dispatch_id,
+    job_run_id: r.job_run_id,
+    item_key: r.item_key,
+    item_preview: r.item_preview,
+    pipeline_id: r.pipeline_id,
+    pipeline_status: r.pipeline_status || 'unknown',
+    workflow_name: r.workflow_name,
+    dispatched_at: toIsoUTC(r.dispatched_at) || r.dispatched_at,
+  }));
+}
+
+/** Stop the sequential drain for this Job — clears next_run_at so the
+ *  scheduler won't pick it up automatically. Does NOT cancel
+ *  pipelines that are already running (caller can do that separately
+ *  on the Pipeline view). Returns true if anything was changed. */
+export function cancelJobDrain(jobId: string): boolean {
+  ensureSchema();
+  const r = db().prepare(`UPDATE jobs SET next_run_at = NULL WHERE id = ?`).run(jobId);
+  return r.changes > 0;
+}

package/lib/jobs/types.ts

@@ -90,6 +90,31 @@ export interface Job {
    *  over to the next tick. Protects disk/RAM from fan-out blow-up. */
   max_per_tick: number;
 
+  /** How this Job paces pipeline dispatch:
+   *
+   *   'sequential' (default) — at most ONE pipeline from this Job runs
+   *                  at a time. Each tick checks whether the previously
+   *                  dispatched pipeline has reached a terminal state;
+   *                  if not, the entire tick is skipped (item stays
+   *                  un-dedup-marked, rolls over to next tick). Avoids
+   *                  hammering downstream systems (GitLab rate limits,
+   *                  Mantis browser-tab races, resource contention).
+   *                  This is the safer default.
+   *
+   *   'parallel'   — each tick dispatches up to max_per_tick items
+   *                  concurrently. Items still hit the global pipeline
+   *                  cap, but no per-Job throttle beyond that. Use
+   *                  only when downstream is known to tolerate burst. */
+  concurrency_mode: 'parallel' | 'sequential';
+
+  /** What to do when an item's pipeline ends in 'failed' state:
+   *    'continue' (default) — proceed to next item; each item is
+   *                independent so one failure doesn't poison the batch.
+   *    'stop'     — halt drain (clears next_run_at). User must Force-
+   *                run again to resume. Use when a failure likely means
+   *                something systemic broke (auth lost, repo gone). */
+  on_failure: 'continue' | 'stop';
+
   last_run_at: string | null;
   next_run_at: string | null;
   created_at: string;
@@ -153,6 +178,12 @@ export interface CreateJobInput {
   /** Per-tick dispatch budget (default 5, capped 1-10 in scheduler). */
   max_per_tick?: number;
 
+  /** 'parallel' (default) | 'sequential'. See Job.concurrency_mode. */
+  concurrency_mode?: 'parallel' | 'sequential';
+
+  /** 'continue' (default) | 'stop'. See Job.on_failure. */
+  on_failure?: 'continue' | 'stop';
+
   /** Default true: first tick records existing items as seen without dispatching. */
   mark_existing_as_seen?: boolean;
 }

package/lib/logger.ts

@@ -6,6 +6,7 @@
 
 import { appendFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
+import { getDataDir } from './dirs';
 
 // Use globalThis to prevent double-init across forge-server.mjs and init.ts
 const loggerKey = Symbol.for('forge-logger-init');
@@ -21,7 +22,6 @@ export function initLogger() {
   let logFile: string | null = null;
   if (!isProduction) {
     try {
-      const { getDataDir } = require('./dirs');
       const dataDir = getDataDir();
       if (!existsSync(dataDir)) mkdirSync(dataDir, { recursive: true });
       logFile = join(dataDir, 'forge.log');

package/lib/notify.ts

@@ -6,11 +6,20 @@ import { loadSettings } from './settings';
 import { addNotification } from './notifications';
 import type { Task } from '@/src/types';
 
+/** Look up the shared pipelineTaskIds Set via globalThis Symbol.
+ *  pipeline.ts populates it on module init; using the Symbol avoids
+ *  a require() that would fire ReferenceError on every task complete
+ *  under concurrent loads (each completion hits notify, 5 pipelines
+ *  × 5 nodes = 25 races per Job run). */
+function isPipelineTask(taskId: string): boolean {
+  const key = Symbol.for('mw-pipeline-task-ids');
+  const set = (globalThis as any)[key] as Set<string> | undefined;
+  return set ? set.has(taskId) : false;
+}
+
 export async function notifyTaskComplete(task: Task) {
   // Skip pipeline tasks
-  let isPipeline = false;
-  try { const { pipelineTaskIds } = require('./pipeline'); isPipeline = pipelineTaskIds.has(task.id); } catch {}
-  if (isPipeline) return;
+  if (isPipelineTask(task.id)) return;
 
   const cost = task.costUSD != null ? `$${task.costUSD.toFixed(4)}` : 'unknown';
   const duration = task.startedAt && task.completedAt
@@ -44,9 +53,7 @@ export async function notifyTaskComplete(task: Task) {
 
 export async function notifyTaskFailed(task: Task) {
   // Skip pipeline tasks
-  let isPipeline = false;
-  try { const { pipelineTaskIds } = require('./pipeline'); isPipeline = pipelineTaskIds.has(task.id); } catch {}
-  if (isPipeline) return;
+  if (isPipelineTask(task.id)) return;
 
   // In-app notification (always)
   try {

package/lib/pipeline-gc.ts

@@ -0,0 +1,105 @@
+/**
+ * Pipeline scratch-dir garbage collector.
+ *
+ * Scans every project's `.forge/worktrees/` for `pipeline-<id>/` dirs
+ * created by the `{{run.tmp_dir}}` mechanism in lib/pipeline.ts. Compares
+ * each dir's pipeline state (status + completedAt) against the retention
+ * settings and rm -rf's expired ones.
+ *
+ * Called from:
+ *  - lib/init.ts setInterval (default every 6h, settings.pipelineTmpGcIntervalHours)
+ *  - cli/mw.ts `forge pipeline gc` (manual / dry-run)
+ *
+ * Retention rules (settings):
+ *  - done   → wiped immediately when pipeline.status flips (in pipeline.ts checkPipelineCompletion).
+ *             GC here only catches done dirs left over from older builds.
+ *  - failed → kept pipelineTmpKeepFailedDays days, then swept.
+ *  - cancelled → kept pipelineTmpKeepCancelledDays days, then swept.
+ *  - running / started → never touched.
+ *  - orphan (no pipeline state file) → swept after 7d based on mtime.
+ */
+
+import { readdirSync, statSync, rmSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { scanProjects } from './projects';
+import { getPipeline } from './pipeline';
+import { loadSettings } from './settings';
+
+export interface GcResult {
+  scanned: number;
+  removed: { path: string; reason: string }[];
+  kept: { path: string; reason: string }[];
+}
+
+const ORPHAN_KEEP_MS = 7 * 86400_000;
+
+export function gcPipelineTmp(opts: { dryRun?: boolean } = {}): GcResult {
+  const settings = loadSettings();
+  const failedKeepMs = Math.max(0, (settings.pipelineTmpKeepFailedDays ?? 3)) * 86400_000;
+  const cancelledKeepMs = Math.max(0, (settings.pipelineTmpKeepCancelledDays ?? 3)) * 86400_000;
+  const cleanDoneNow = settings.pipelineTmpCleanDoneImmediate !== false;
+  const now = Date.now();
+
+  const removed: GcResult['removed'] = [];
+  const kept: GcResult['kept'] = [];
+  let scanned = 0;
+
+  for (const proj of scanProjects()) {
+    const wtDir = join(proj.path, '.forge', 'worktrees');
+    if (!existsSync(wtDir)) continue;
+    let entries: string[];
+    try { entries = readdirSync(wtDir); } catch { continue; }
+
+    for (const entry of entries) {
+      if (!entry.startsWith('pipeline-')) continue;
+      const pipeId = entry.slice('pipeline-'.length);
+      const fullPath = join(wtDir, entry);
+      scanned++;
+
+      const pipeline = getPipeline(pipeId);
+
+      // Orphan: pipeline state gone. Use mtime as best signal.
+      if (!pipeline) {
+        let mtimeMs: number;
+        try { mtimeMs = statSync(fullPath).mtimeMs; } catch { continue; }
+        if (now - mtimeMs > ORPHAN_KEEP_MS) {
+          if (!opts.dryRun) {
+            try { rmSync(fullPath, { recursive: true, force: true }); } catch {}
+          }
+          removed.push({ path: fullPath, reason: `orphan (>${Math.round(ORPHAN_KEEP_MS / 86400_000)}d)` });
+        } else {
+          kept.push({ path: fullPath, reason: 'orphan, still fresh' });
+        }
+        continue;
+      }
+
+      const completedAt = pipeline.completedAt ? Date.parse(pipeline.completedAt) : null;
+      let shouldRemove = false;
+      let reason = '';
+
+      if (pipeline.status === 'done' && cleanDoneNow) {
+        shouldRemove = true;
+        reason = 'done (immediate cleanup enabled)';
+      } else if (pipeline.status === 'failed' && completedAt && now - completedAt > failedKeepMs) {
+        shouldRemove = true;
+        reason = `failed > ${settings.pipelineTmpKeepFailedDays}d`;
+      } else if (pipeline.status === 'cancelled' && completedAt && now - completedAt > cancelledKeepMs) {
+        shouldRemove = true;
+        reason = `cancelled > ${settings.pipelineTmpKeepCancelledDays}d`;
+      }
+
+      if (shouldRemove) {
+        if (!opts.dryRun) {
+          try { rmSync(fullPath, { recursive: true, force: true }); } catch (e) {
+            console.warn(`[pipeline-gc] rm ${fullPath} failed: ${(e as Error).message}`);
+          }
+        }
+        removed.push({ path: fullPath, reason });
+      } else {
+        kept.push({ path: fullPath, reason: `status=${pipeline.status}` });
+      }
+    }
+  }
+
+  return { scanned, removed, kept };
+}