nemoris 0.1.0

package/src/runtime/daemon.js

@@ -0,0 +1,729 @@
+import fs from "node:fs";
+import path from "node:path";
+import { OpenClawShadowBridge } from "../shadow/bridge.js";
+import { Scheduler } from "./scheduler.js";
+import { Executor } from "./executor.js";
+import { SchedulerStateStore } from "./scheduler-state.js";
+import { computeNextRun, computeRunSlot, formatRunState, isDue, iso } from "./schedule.js";
+import { DeliveryManager } from "./delivery-manager.js";
+import { createRuntimeId } from "../utils/ids.js";
+import { routeJob } from "./route-job.js";
+import { HealingTick } from "../daemon/healing-tick.js";
+import { RepairLog } from "../daemon/repair-log.js";
+import { RuleStaging } from "../daemon/rule-staging.js";
+
+export const DAEMON_LOCK_FILE = "daemon.lock";
+
+function buildIdempotencyInstance(job, slotTime) {
+  if (!job.idempotencyKey) return null;
+  return `${job.idempotencyKey}:${iso(slotTime)}`;
+}
+
+function buildLeaseId(jobId, now) {
+  return `${jobId}:${now.getTime()}:${createRuntimeId("lease")}`;
+}
+
+function buildRetryDelay(attempt) {
+  return Math.min(Math.max(attempt, 1), 5) * 60 * 1000;
+}
+
+function isStaleInProgress(entry, now, maxRuntimeSeconds) {
+  if (!entry.inProgressStartedAt) return true;
+  const startedAt = new Date(entry.inProgressStartedAt).getTime();
+  const leaseMs = Math.max((maxRuntimeSeconds || 30) * 2000, 60_000);
+  return startedAt + leaseMs <= now.getTime();
+}
+
+function normalizeFatalError(error) {
+  if (error instanceof Error) {
+    return error.stack || error.message;
+  }
+  if (typeof error === "string") return error;
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return String(error);
+  }
+}
+
+function isMissingProcessError(error) {
+  const message = String(error?.message || "");
+  return error?.code === "ESRCH" ||
+    error?.code === "ENOENT" ||
+    /no such process/i.test(message) ||
+    /process does not exist/i.test(message) ||
+    /could not find/i.test(message);
+}
+
+function readPidFromFile(filePath, fsImpl = fs) {
+  try {
+    const pid = Number.parseInt(fsImpl.readFileSync(filePath, "utf8").trim(), 10);
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function safeUnlink(filePath, fsImpl = fs) {
+  try {
+    fsImpl.unlinkSync(filePath);
+  } catch {
+    // Best effort cleanup.
+  }
+}
+
+function removeOwnedFile(filePath, pid, fsImpl = fs) {
+  try {
+    if (readPidFromFile(filePath, fsImpl) === pid) {
+      fsImpl.unlinkSync(filePath);
+    }
+  } catch {
+    // Best effort cleanup.
+  }
+}
+
+export function isProcessRunning(pid, { processImpl = process } = {}) {
+  if (!Number.isInteger(pid) || pid <= 0) {
+    return false;
+  }
+
+  try {
+    processImpl.kill(pid, 0);
+    return true;
+  } catch (error) {
+    if (isMissingProcessError(error)) {
+      return false;
+    }
+    return true;
+  }
+}
+
+function emitLockHeldError({ lockFile, pidFile, fsImpl = fs, processImpl = process, stderr = process.stderr }) {
+  const pid = readPidFromFile(pidFile, fsImpl) ?? readPidFromFile(lockFile, fsImpl);
+  const suffix = pid ? ` (pid ${pid})` : "";
+  stderr.write(`Another daemon is already running${suffix}. Stop it first with: nemoris stop\n`);
+  processImpl.exit(1);
+  return null;
+}
+
+export function acquireDaemonLifecycleLock({
+  stateDir,
+  pidFile = path.join(stateDir, "daemon.pid"),
+  fsImpl = fs,
+  processImpl = process,
+  stderr = process.stderr,
+} = {}) {
+  const lockFile = path.join(stateDir, DAEMON_LOCK_FILE);
+  const pid = processImpl.pid;
+  fsImpl.mkdirSync(stateDir, { recursive: true });
+
+  let released = false;
+  let lockFd = null;
+  let usesFlock = typeof fsImpl.flockSync === "function";
+
+  const release = () => {
+    if (released) return;
+    released = true;
+
+    removeOwnedFile(pidFile, pid, fsImpl);
+
+    if (usesFlock) {
+      if (lockFd !== null) {
+        try {
+          fsImpl.flockSync(lockFd, "un");
+        } catch {
+          // Ignore lock release failures during shutdown.
+        }
+        try {
+          fsImpl.closeSync(lockFd);
+        } catch {
+          // Ignore fd close failures during shutdown.
+        }
+        lockFd = null;
+      }
+      return;
+    }
+
+    removeOwnedFile(lockFile, pid, fsImpl);
+  };
+
+  if (usesFlock) {
+    try {
+      lockFd = fsImpl.openSync(lockFile, "w");
+      fsImpl.flockSync(lockFd, "ex_nb");
+      fsImpl.writeFileSync(lockFd, `${pid}\n`, "utf8");
+      fsImpl.writeFileSync(pidFile, `${pid}\n`, "utf8");
+      return { lockFile, pidFile, release };
+    } catch (error) {
+      if (lockFd !== null) {
+        try {
+          fsImpl.closeSync(lockFd);
+        } catch {
+          // Ignore close failures on failed lock acquisition.
+        }
+        lockFd = null;
+      }
+      if (error?.code === "EWOULDBLOCK" || error?.code === "EAGAIN") {
+        return emitLockHeldError({ lockFile, pidFile, fsImpl, processImpl, stderr });
+      }
+      throw error;
+    }
+  }
+
+  usesFlock = false;
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      const fd = fsImpl.openSync(lockFile, "wx");
+      try {
+        fsImpl.writeFileSync(fd, `${pid}\n`, "utf8");
+      } finally {
+        fsImpl.closeSync(fd);
+      }
+      fsImpl.writeFileSync(pidFile, `${pid}\n`, "utf8");
+      return { lockFile, pidFile, release };
+    } catch (error) {
+      if (error?.code !== "EEXIST") {
+        throw error;
+      }
+
+      const lockPid = readPidFromFile(lockFile, fsImpl) ?? readPidFromFile(pidFile, fsImpl);
+      if (isProcessRunning(lockPid, { processImpl })) {
+        return emitLockHeldError({ lockFile, pidFile, fsImpl, processImpl, stderr });
+      }
+
+      safeUnlink(lockFile, fsImpl);
+    }
+  }
+
+  return emitLockHeldError({ lockFile, pidFile, fsImpl, processImpl, stderr });
+}
+
+export class SchedulerDaemon {
+  constructor({ projectRoot, liveRoot, stateRoot, fetchImpl, deliveryAck, taskContract, telegramSendFn, envelopeStore } = {}) {
+    this.projectRoot = projectRoot;
+    this.liveRoot = liveRoot;
+    this.stateRoot = stateRoot;
+    this.scheduler = new Scheduler({ projectRoot, liveRoot, stateRoot });
+    this.executor = new Executor({ projectRoot, liveRoot, stateRoot, fetchImpl });
+    this.stateStore = new SchedulerStateStore({ rootDir: path.join(stateRoot, "scheduler") });
+    this.deliveryManager = new DeliveryManager({ projectRoot, liveRoot, stateRoot, fetchImpl, stateStore: this.stateStore });
+    this.deliveryAck = deliveryAck || null;
+    this.taskContract = taskContract || null;
+    this.telegramSendFn = telegramSendFn || null;
+    this.envelopeStore = envelopeStore || null;
+    this.maintenanceCycleCount = 0;
+    this.shuttingDown = false;
+    this.healingTick = null; // Initialized on first maintenance run
+    this.shutdownRequestedAt = null;
+    this.activeRuns = 0;
+    this.lastTickResult = null;
+    this.orchestrator = null;
+    this.mcpConsumer = null;
+    this.repairLog = null;
+    this.ruleStaging = null;
+    this._globalHandlersInstalled = false;
+    this._fatalExitInProgress = false;
+  }
+
+  setOrchestrator(orchestrator) {
+    this.orchestrator = orchestrator;
+  }
+
+  setMcpConsumer(consumer) {
+    this.mcpConsumer = consumer;
+  }
+
+  _collectOpenSqliteHandles() {
+    const handles = [];
+    const seen = new Set();
+    const addHandle = (db, label) => {
+      if (!db || typeof db.exec !== "function" || seen.has(db)) return;
+      seen.add(db);
+      handles.push({ db, label });
+    };
+
+    addHandle(this.stateStore?.db, "stateStore");
+    addHandle(this.taskContract?.db, "taskContract");
+    addHandle(this.deliveryAck?.db, "deliveryAck");
+    addHandle(this.envelopeStore?.db, "envelopeStore");
+    addHandle(this.repairLog?.db, "repairLog");
+    addHandle(this.ruleStaging?.db, "ruleStaging");
+
+    const sqliteStores = this.scheduler?.memoryStore?.sqliteStores;
+    if (sqliteStores instanceof Map) {
+      for (const [dbPath, store] of sqliteStores.entries()) {
+        addHandle(store?.db, `memoryStore:${dbPath}`);
+      }
+    }
+
+    return handles;
+  }
+
+  checkpointOpenDbs() {
+    const results = [];
+    for (const { db, label } of this._collectOpenSqliteHandles()) {
+      try {
+        db.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+        results.push({ label, ok: true });
+      } catch (error) {
+        results.push({ label, ok: false, error: error.message });
+      }
+    }
+    return results;
+  }
+
+  _handleGlobalFailure(event, error) {
+    if (this._fatalExitInProgress) return;
+    this._fatalExitInProgress = true;
+
+    const normalizedError = normalizeFatalError(error);
+    console.error(JSON.stringify({ service: "scheduler_daemon", event, error: normalizedError }));
+
+    try {
+      this.repairLog?.write?.({
+        source: "scheduler_daemon",
+        type: "unhandled_crash",
+        action: "restart",
+        result: "escalated",
+        severity: "critical",
+        diagnosis: normalizedError,
+        context: JSON.stringify({ event }),
+        escalated: 1,
+      });
+    } catch {
+      // Best-effort only on fatal path.
+    }
+
+    this.checkpointOpenDbs();
+    process.exit(1);
+  }
+
+  installGlobalHandlers() {
+    if (this._globalHandlersInstalled) return;
+
+    if (!this.repairLog && this.stateStore?.db) {
+      this.repairLog = new RepairLog(this.stateStore.db);
+      this.repairLog.ensureSchema();
+    }
+
+    this._globalHandlersInstalled = true;
+    this._onUnhandledRejection = (reason) => this._handleGlobalFailure("unhandledRejection", reason);
+    this._onUncaughtException = (error) => this._handleGlobalFailure("uncaughtException", error);
+    process.on("unhandledRejection", this._onUnhandledRejection);
+    process.on("uncaughtException", this._onUncaughtException);
+  }
+
+  async checkMigrationBanner() {
+    if (this._bannerChecked || !this.liveRoot) return;
+    this._bannerChecked = true;
+    try {
+      const bridge = new OpenClawShadowBridge({ liveRoot: this.liveRoot });
+      const config = await bridge.loadConfig();
+      if (config.agents?.list?.length > 0) {
+        console.log(`[nemoris] OpenClaw detected at ${this.liveRoot} — /transfer to migrate`);
+      }
+    } catch { /* non-blocking */ }
+  }
+
+  async listDueJobs(now = new Date()) {
+    const [plans, state] = await Promise.all([this.scheduler.listJobPlans(), this.stateStore.load()]);
+    return plans.filter((job) => {
+      const current = state.jobs[job.id] || {};
+      const slotTime = computeRunSlot(job.trigger, now);
+      const idempotencyInstance = buildIdempotencyInstance(job, slotTime);
+      if (current.inProgress && !isStaleInProgress(current, now, job.budget?.maxRuntimeSeconds)) {
+        return false;
+      }
+      if (idempotencyInstance && current.lastIdempotencyKey === idempotencyInstance && current.lastStatus === "ok") {
+        return false;
+      }
+      return isDue(job.trigger, current.lastRunAt, current.nextRunAt, now);
+    });
+  }
+
+  async tick({ now = new Date(), mode = "dry-run" } = {}) {
+    await this.checkMigrationBanner();
+    if (this.shuttingDown) {
+      const earlyResult = {
+        timestamp: iso(now),
+        maxConcurrentJobs: 0,
+        maxJobsPerTick: 0,
+        dueCount: 0,
+        results: []
+      };
+      this.lastTickResult = earlyResult;
+      return earlyResult;
+    }
+
+    const runtime = await this.scheduler.loadRuntime();
+    const maxJobsPerTick = runtime.runtime?.concurrency?.maxJobsPerTick ?? runtime.runtime?.concurrency?.maxConcurrentJobs ?? 2;
+    const dueJobs = (await this.listDueJobs(now)).slice(0, maxJobsPerTick);
+    const results = [];
+    const maintenance = await this.runMaintenance(runtime, now);
+    const allDueJobs = await this.listDueJobs(now);
+    const deferredCount = Math.max(allDueJobs.length - dueJobs.length, 0);
+
+    for (const job of dueJobs) {
+      const slotTime = computeRunSlot(job.trigger, now);
+      const idempotencyInstance = buildIdempotencyInstance(job, slotTime);
+      const leaseId = buildLeaseId(job.id, now);
+      const startedState = await this.stateStore.mutateJob(job.id, (current) => {
+        if (current.inProgress && !isStaleInProgress(current, now, job.budget?.maxRuntimeSeconds)) {
+          return current;
+        }
+        if (idempotencyInstance && current.lastIdempotencyKey === idempotencyInstance && current.lastStatus === "ok") {
+          return current;
+        }
+        if (!isDue(job.trigger, current.lastRunAt, current.nextRunAt, now)) {
+          return current;
+        }
+        const retrySlotKey = current.retrySlotKey || null;
+        const retryAttempt = retrySlotKey === idempotencyInstance ? current.retryAttempt || 0 : 0;
+        return {
+          ...current,
+          inProgress: true,
+          inProgressStartedAt: iso(now),
+          currentIdempotencyKey: idempotencyInstance,
+          currentLeaseId: leaseId,
+          retrySlotKey: retrySlotKey === idempotencyInstance ? retrySlotKey : current.retrySlotKey || null,
+          retryAttempt
+        };
+      });
+
+      if (!startedState.inProgress || startedState.currentLeaseId !== leaseId) {
+        results.push({
+          jobId: job.id,
+          status: "skipped",
+          reason: startedState.inProgress ? "already_claimed" : "not_due",
+          state: formatRunState(startedState)
+        });
+        continue;
+      }
+
+      if (this.shuttingDown) {
+        const jobState = await this.stateStore.updateJob(job.id, {
+          inProgress: false,
+          inProgressStartedAt: null,
+          currentIdempotencyKey: null,
+          currentLeaseId: null,
+          abandonedAt: iso(now),
+          abandonedReason: "shutdown_requested_before_execution"
+        });
+        results.push({
+          jobId: job.id,
+          status: "abandoned",
+          reason: "shutdown_requested_before_execution",
+          state: formatRunState(jobState)
+        });
+        continue;
+      }
+
+      try {
+        this.activeRuns += 1;
+        const run = await routeJob({
+          jobId: job.id,
+          mode,
+          orchestrator: this.orchestrator,
+          executor: this.executor,
+          skipOrchestrator: job.skipOrchestrator || false,
+          shadowImport: true,
+          idempotencyKey: idempotencyInstance,
+        });
+        const nextRunAt = computeNextRun(job.trigger, now);
+        const jobState = await this.stateStore.updateJob(job.id, {
+          lastRunAt: iso(now),
+          nextRunAt: iso(nextRunAt),
+          lastStatus: "ok",
+          lastRunFile: run.filePath,
+          lastError: null,
+          consecutiveFailures: 0,
+          inProgress: false,
+          inProgressStartedAt: null,
+          currentIdempotencyKey: null,
+          currentLeaseId: null,
+          lastIdempotencyKey: idempotencyInstance,
+          retryAttempt: 0,
+          retrySlotKey: null
+        });
+        results.push({
+          jobId: job.id,
+          status: "ok",
+          runFile: run.filePath,
+          notificationFiles: run.notificationFiles || [],
+          state: formatRunState(jobState)
+        });
+      } catch (error) {
+        const existing = (await this.stateStore.load()).jobs[job.id] || {};
+        const currentAttempt =
+          existing.retrySlotKey === idempotencyInstance ? (existing.retryAttempt || 0) + 1 : 1;
+        const maxAttempts = job.retry?.maxAttempts ?? 1;
+        const retryScheduled = currentAttempt < maxAttempts;
+        const nextRunAt = retryScheduled
+          ? new Date(now.getTime() + buildRetryDelay(currentAttempt))
+          : computeNextRun(job.trigger, now);
+        const jobState = await this.stateStore.updateJob(job.id, {
+          lastRunAt: iso(now),
+          nextRunAt: iso(nextRunAt),
+          lastStatus: "error",
+          lastError: error.message,
+          consecutiveFailures: (existing.consecutiveFailures || 0) + 1,
+          inProgress: false,
+          inProgressStartedAt: null,
+          currentIdempotencyKey: null,
+          currentLeaseId: null,
+          retryAttempt: retryScheduled ? currentAttempt : 0,
+          retrySlotKey: retryScheduled ? idempotencyInstance : null
+        });
+        results.push({
+          jobId: job.id,
+          status: "error",
+          error: error.message,
+          retryScheduled,
+          retryAttempt: currentAttempt,
+          runFile: error.runFile || null,
+          state: formatRunState(jobState)
+        });
+      } finally {
+        this.activeRuns = Math.max(0, this.activeRuns - 1);
+      }
+    }
+
+    const tickResult = {
+      timestamp: iso(now),
+      maxConcurrentJobs: maxJobsPerTick,
+      maxJobsPerTick,
+      executionMode: "sequential",
+      selectedJobCount: dueJobs.length,
+      executedJobCount: results.filter((item) => item.status !== "skipped").length,
+      deferredJobCount: deferredCount,
+      sequentialExecution: true,
+      dueCount: dueJobs.length,
+      maintenance,
+      results
+    };
+    this.lastTickResult = tickResult;
+    return tickResult;
+  }
+
+  getLastTickResult() {
+    return this.lastTickResult;
+  }
+
+  async runMaintenance(runtime, now = new Date()) {
+    const config = runtime.runtime?.maintenance || {};
+    const walThresholdBytes = Number(config.walCheckpointThresholdBytes ?? 64 * 1024 * 1024);
+    const pruneOnTick = config.pruneOnTick ?? true;
+    const sweepHandoffsOnTick = config.sweepPendingHandoffsOnTick ?? true;
+    const sweepFollowUpsOnTick = config.sweepPendingFollowUpsOnTick ?? true;
+    const agents = Object.keys(runtime.agents || {});
+    const wal = [];
+    for (const agentId of agents) {
+      try {
+        const result = await this.scheduler.memoryStore.manageSqlite(agentId, {
+          thresholdBytes: walThresholdBytes
+        });
+        wal.push({ agentId, ...result });
+      } catch (error) {
+        wal.push({ agentId, action: "error", error: error.message });
+      }
+    }
+
+    const handoffs = sweepHandoffsOnTick
+      ? await this.deliveryManager.sweepPendingHandoffs({
+          now,
+          timeoutMinutes: runtime.runtime?.handoffs?.pendingTimeoutMinutes,
+          escalateOnExpiry: runtime.runtime?.handoffs?.escalateOnExpiry,
+          escalationDeliveryProfile: runtime.runtime?.handoffs?.escalationDeliveryProfile
+        })
+      : { timestamp: iso(now), pendingCount: 0, expiredCount: 0, results: [] };
+
+    const followUps = sweepFollowUpsOnTick
+      ? await this.deliveryManager.sweepPendingFollowUps({
+          now,
+          timeoutMinutes: runtime.runtime?.followUps?.pendingTimeoutMinutes,
+          maxFollowUpDepth: runtime.runtime?.followUps?.maxFollowUpDepth,
+          escalateOnExpiry: runtime.runtime?.followUps?.escalateOnExpiry,
+          escalationDeliveryProfile: runtime.runtime?.followUps?.escalationDeliveryProfile
+        })
+      : { timestamp: iso(now), pendingCount: 0, expiredCount: 0, results: [] };
+
+    // Purge expired follow-up completion records
+    try {
+      await this.stateStore.ensureReady();
+      this.stateStore.purgeExpiredFollowUps(now.toISOString());
+    } catch (_err) {
+      // Non-fatal — log and continue
+    }
+
+    // Self-healing maintenance
+    let healing = { expiredRules: 0, prunedEntries: 0, promoted: 0 };
+    try {
+      if (!this.healingTick) {
+        await this.stateStore.ensureReady();
+        const repairLog = new RepairLog(this.stateStore.db);
+        repairLog.ensureSchema();
+        const ruleStaging = new RuleStaging(this.stateStore.db);
+        ruleStaging.ensureSchema();
+        this.repairLog = repairLog;
+        this.ruleStaging = ruleStaging;
+        this.healingTick = new HealingTick({
+          repairLog,
+          ruleStaging,
+          executeAction: async (action, context) => {
+            // Action execution is wired in Chunk 8 (action handlers)
+            console.log(JSON.stringify({ service: "healing_daemon", event: "action_executed", action, context, timestamp: new Date().toISOString() }));
+            return true;
+          },
+          notify: this.telegramSendFn
+            ? async (severity, message) => { if (severity !== "silent") await this.telegramSendFn(message); }
+            : async () => {}
+        });
+      }
+      healing = this.healingTick.runMaintenanceSweep();
+    } catch (err) {
+      console.log(JSON.stringify({ service: "healing_daemon", event: "maintenance_error", error: err.message, timestamp: new Date().toISOString() }));
+    }
+
+    const pruning = pruneOnTick
+      ? await Promise.all([
+          this.executor.runStore.prune(),
+          this.deliveryManager.notificationStore.prune(),
+          this.deliveryManager.deliveryStore.prune()
+        ])
+      : [];
+
+    // Sweep pending decisions (Pillar 2)
+    let pendingDecisions = { rePinged: 0, parked: 0 };
+    if (this.deliveryAck) {
+      try {
+        pendingDecisions = await this.deliveryAck.sweepPendingDecisions({
+          now,
+          sendFn: this.telegramSendFn || (() => {}),
+        });
+      } catch (_err) { /* non-fatal */ }
+    }
+
+    // Sweep timed-out tasks (Pillar 3)
+    let taskSweep = { timedOut: 0 };
+    if (this.taskContract) {
+      try {
+        const overdue = this.taskContract.sweepTimedOutTasks();
+        taskSweep = { timedOut: overdue.length };
+      } catch (_err) { /* non-fatal */ }
+    }
+
+    // Prune old envelopes (30-day retention)
+    let envelopePruned = 0;
+    if (this.envelopeStore) {
+      try {
+        envelopePruned = this.envelopeStore.prune(30);
+      } catch (_err) { /* non-fatal */ }
+    }
+
+    // Embedding maintenance — run every 5th cycle to avoid overhead
+    this.maintenanceCycleCount += 1;
+    let embeddingMaintenance = { ran: false, cycle: this.maintenanceCycleCount };
+    if (this.maintenanceCycleCount % 5 === 0 && this.scheduler.embeddingIndex) {
+      const staleAgents = [];
+      for (const agentId of agents) {
+        try {
+          const health = await this.scheduler.memoryStore.getEmbeddingHealth(agentId);
+          const staleCount = health.embeddingHealth?.staleCount || 0;
+          const missingCount = health.embeddingHealth?.missingCount || 0;
+          if (staleCount > 0 || missingCount > 0) {
+            staleAgents.push({ agentId, staleCount, missingCount });
+          }
+        } catch { /* non-fatal */ }
+      }
+      if (staleAgents.length > 0) {
+        // Fire-and-forget background rebuilds
+        for (const { agentId } of staleAgents) {
+          this.scheduler.memoryStore
+            .rebuildEmbeddings(agentId, { embeddingIndex: this.scheduler.embeddingIndex })
+            .catch((err) => {
+              console.warn(`[nemoris] embedding rebuild failed for ${agentId}: ${err?.message || err}`);
+            });
+        }
+      }
+      embeddingMaintenance = { ran: true, cycle: this.maintenanceCycleCount, staleAgents };
+    }
+
+    const state = {
+      timestamp: iso(now),
+      wal,
+      handoffs,
+      followUps,
+      pruning,
+      pendingDecisions,
+      taskSweep,
+      envelopePruned,
+      embeddingMaintenance,
+      healing
+    };
+    await this.stateStore.setMeta("maintenance", state);
+    return state;
+  }
+
+  requestShutdown(now = new Date()) {
+    this.shuttingDown = true;
+    this.shutdownRequestedAt = iso(now);
+  }
+
+  async waitForIdle(timeoutMs = 15000) {
+    const deadline = Date.now() + timeoutMs;
+    while (this.activeRuns > 0 && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+    return {
+      idle: this.activeRuns === 0,
+      activeRuns: this.activeRuns
+    };
+  }
+
+  async close(options = {}) {
+    this.requestShutdown();
+    await this.waitForIdle(options.drainTimeoutMs || 15000);
+    if (this.mcpConsumer) {
+      try { await this.mcpConsumer.shutdown(); } catch { /* best-effort */ }
+    }
+    this.stateStore.close();
+  }
+}
+
+/**
+ * Single pass of the interactive drain loop. Reads all queued interactive jobs,
+ * marks them running, executes each via the callback, and marks succeeded/failed.
+ *
+ * Exported for testability. The daemon's interactiveDrainLoop() calls this
+ * in a loop with a 100ms sleep between passes.
+ *
+ * @param {import('./scheduler-state.js').SchedulerStateStore} stateStore
+ * @param {(job: object) => Promise<object>} processJob
+ * @returns {Promise<Array<{jobId: string, status: string, error?: string}>>}
+ */
+export async function drainInteractiveOnce(stateStore, processJob, options = {}) {
+  const jobs = stateStore.drainQueuedInteractiveJobs();
+  const results = [];
+
+  for (const job of jobs) {
+    try {
+      const outcome = await processJob(job);
+      if (outcome?.requeue) {
+        stateStore.updateInteractiveJobStatus(job.job_id, "queued");
+        results.push({ jobId: job.job_id, status: "requeued", reason: outcome.reason || "requeued" });
+        continue;
+      }
+      stateStore.updateInteractiveJobStatus(job.job_id, "succeeded");
+      results.push({ jobId: job.job_id, status: "succeeded" });
+    } catch (error) {
+      console.error(JSON.stringify({ service: "interactive_drain", jobId: job.job_id, error: error.message, stack: error.stack?.split("\n").slice(0, 5) }));
+      stateStore.updateInteractiveJobStatus(job.job_id, "failed");
+      results.push({ jobId: job.job_id, status: "failed", error: error.message });
+      if (options.onJobFailure) {
+        try { await options.onJobFailure(job, error); } catch (_) {}
+      }
+    }
+  }
+
+  return results;
+}