@humwick/server 0.130.1 → 0.130.2

package/src/db/timer-ingress.ts

@@ -104,13 +104,18 @@ function rowToIntent(row: IntentDbRow): TimerIngressIntentRow {
   };
 }
 
+const installedOn = new WeakSet<object>();
+
 /** Idempotent, crash-safe schema install (one transaction; re-running over existing data is a
  *  no-op). Bootstrap DDL — it runs ONCE at startup as its own committed statement, never lazily
  *  inside an op, so a rolled-back op can never undo the schema it ran against (§4 allowlist). */
 export function installTimerIngressSchema(
-  options: { requestKeyMigrationFaults?: TimerIngressRequestKeyMigrationFaults } = {},
+  options: { requestKeyMigrationFaults?: TimerIngressRequestKeyMigrationFaults; ifNotInstalled?: boolean } = {},
 ): void {
   const db = getDb();
+  // #1870 S7: `startServer` installs it before bind (the boot exception); `createTimerIngress` passes
+  // `ifNotInstalled` so its later call runs no DDL on the main connection. Explicit calls always install.
+  if (options.ifNotInstalled && installedOn.has(db)) return;
   db.transaction(() => {
     // AUTOINCREMENT is deliberate: `seq` must be monotone and NEVER reused (the watermark's meaning
     // depends on it — and it doubles as the per-timer apply FENCE), even if settled rows are pruned.
@@ -165,6 +170,7 @@ export function installTimerIngressSchema(
     db.run("INSERT OR IGNORE INTO timer_ingress_watermark (id, applied_seq, updated_at) VALUES (1, 0, 0)");
     options.requestKeyMigrationFaults?.beforeCommit?.();
   }).immediate();
+  installedOn.add(db);
 }
 
 // ── Reads ───────────────────────────────────────────────────────────────────────────────────────

package/src/db/token-db-write-ops.ts

@@ -25,6 +25,7 @@ import {
   type PruneExpiredTokenRecordsInput,
 } from "../token-db.ts";
 import type { TokenRecord } from "../types.ts";
+import { recordLaunchFact, type LaunchFact } from "./launch-facts.ts";
 import type { TokenRevokeReason } from "../security.ts";
 import type { WriterOpContext, WriterOpSpec } from "./write-ops.ts";
 
@@ -41,6 +42,11 @@ export interface HandoffTokenOpArgs {
   readonly newJti: string;
 }
 
+/** Args for `tokens.createRunner`: a runner mint plus, for a fresh launch, its launch fact (#1870 S7). */
+export interface CreateRunnerTokenOpArgs extends CreateTokenOpArgs {
+  readonly launch?: { readonly agentId: string; readonly fact: LaunchFact };
+}
+
 export interface RevokeTokenOpArgs {
   readonly jti: string;
   readonly reason?: TokenRevokeReason;
@@ -72,6 +78,17 @@ export const TOKEN_DB_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object
     fn: (args: CreateTokenOpArgs, ctx: WriterOpContext) => createTokenInternal(args.input, args.jti, ctx.now),
     serializeIdempotencyResult: stripCreateToken,
   },
+  // A runner mint and the launch fact its successors derive from, in one transaction.
+  "tokens.createRunner": {
+    offLoop: true,
+    idempotent: false,
+    fn: (args: CreateRunnerTokenOpArgs, ctx: WriterOpContext) => {
+      const issued = createTokenInternal(args.input, args.jti, ctx.now);
+      if (args.launch) recordLaunchFact(args.launch.agentId, args.launch.fact, ctx.now);
+      return issued;
+    },
+    serializeIdempotencyResult: stripCreateToken,
+  },
   // Non-idempotent (mints and links a successor); folds the eligibility read (ADR §3.4).
   "tokens.handoff": {
     offLoop: true,

package/src/db/token-standing.ts

@@ -61,24 +61,28 @@ export function tokenRevocationReason(jti: string): string | null | undefined {
 /** A rotation retires one token for its successor, so it is never a lineage revocation. */
 export type LineageRevokeReason = Exclude<TokenRevokeReason, "rotation">;
 
-/** #1245 — a token and every token handed off from it (token-db.ts handoffTokenInternal) are one credential, so
- * revoking any of them revokes the whole lineage: unretired predecessors, pending successors and their renewals.
- * A hard reason also hardens stale members, so no auth, renewal or orchestrator recovery reaches the lineage. A
- * stale walk stops at stale members: what follows them is an orchestrator recovery. It never enters a `keep` token. */
-export function revokeTokenLineage(seeds: readonly string[], reason: LineageRevokeReason | null, nowSeconds: number, keep: readonly string[] = []): string[] {
-  if (seeds.length === 0) return [];
-  const rows = getDb().query(`
+// #3776 — each step must be a key lookup whatever sqlite_stat1 says: CROSS JOIN pins the join order and the unary
+// `+` keeps the predecessor index off the backward step, so stale stats cannot make the walk scan per member.
+export const TOKEN_LINEAGE_REVOKE_SQL = `
     WITH RECURSIVE kept(jti) AS (SELECT value FROM json_each(?1)),
     lineage(jti) AS (
       SELECT value FROM json_each(?2) WHERE value NOT IN kept
-      UNION SELECT t.jti FROM lineage l JOIN tokens p ON p.jti = l.jti JOIN tokens t ON t.predecessor_jti = p.jti
+      UNION SELECT t.jti FROM lineage l CROSS JOIN tokens p ON p.jti = l.jti CROSS JOIN tokens t ON t.predecessor_jti = p.jti
         WHERE t.jti NOT IN kept AND (?4 IS NOT 'stale' OR p.revoked_reason IS NOT 'stale')
-      UNION SELECT p.predecessor_jti FROM lineage l JOIN tokens p ON p.jti = l.jti
-        WHERE p.predecessor_jti IS NOT NULL AND p.predecessor_jti NOT IN kept AND (?4 IS NOT 'stale' OR p.revoked_reason IS NOT 'stale')
+      UNION SELECT p.predecessor_jti FROM lineage l CROSS JOIN tokens p ON p.jti = l.jti
+        WHERE +p.predecessor_jti IS NOT NULL AND +p.predecessor_jti NOT IN kept AND (?4 IS NOT 'stale' OR p.revoked_reason IS NOT 'stale')
     )
     UPDATE tokens SET revoked_at = COALESCE(revoked_at, ?3), revoked_reason = ?4
     WHERE jti IN lineage AND (revoked_at IS NULL OR (?4 IS NOT 'stale' AND revoked_reason = 'stale'))
     RETURNING jti
-  `).all(JSON.stringify(keep), JSON.stringify(seeds), nowSeconds, reason) as Array<{ jti: string }>;
+  `;
+
+/** #1245 — a token and every token handed off from it (token-db.ts handoffTokenInternal) are one credential, so
+ * revoking any of them revokes the whole lineage: unretired predecessors, pending successors and their renewals.
+ * A hard reason also hardens stale members, so no auth, renewal or orchestrator recovery reaches the lineage. A
+ * stale walk stops at stale members: what follows them is an orchestrator recovery. It never enters a `keep` token. */
+export function revokeTokenLineage(seeds: readonly string[], reason: LineageRevokeReason | null, nowSeconds: number, keep: readonly string[] = []): string[] {
+  if (seeds.length === 0) return [];
+  const rows = getDb().query(TOKEN_LINEAGE_REVOKE_SQL).all(JSON.stringify(keep), JSON.stringify(seeds), nowSeconds, reason) as Array<{ jti: string }>;
   return rows.map((row) => row.jti);
 }

package/src/db/workflow-contact.ts

@@ -20,7 +20,7 @@ export interface WorkflowHeldContact { runId: string; reason: WorkflowContactRea
 
 /** Notices that bypass the step-contact hold: the run's own coordinator notices (including GO awaiting an external
  *  hold's release), and a step agent held on a provider block, which waits until someone acts (#3647). */
-const COORDINATOR_NOTICE_KINDS = new Set(["workflow.started", "workflow.landed", "workflow.escalated", "workflow.result", "workflow.review-passed", "agent.spawn_failed", "agent.spawn_recovered", "delegation.budget_exhausted", "agent.provider_blocked", "agent.provider_still_held"]);
+const COORDINATOR_NOTICE_KINDS = new Set(["workflow.started", "workflow.landed", "workflow.escalated", "workflow.result", "workflow.review-passed", "workflow.paused", "workflow.unpaused", "agent.spawn_failed", "agent.spawn_recovered", "delegation.budget_exhausted", "agent.provider_blocked", "agent.provider_still_held"]);
 export const WORKFLOW_SCOPE = "local";
 export const WORKFLOW_RUN_TABLE = "kernel_entity__workflow__run";
 const LINK_TABLE = "kernel_entity__workflow__link";

package/src/db/workflow-task-read-ops.ts

@@ -185,3 +185,33 @@ export function openWorkflowRunHolders(): { teamIds: Set<string>; coordinatorIds
   }
   return holders;
 }
+
+/** #3757 — a running run a coordinator paused, with its input Task's team; `pause` is the stored JSON. */
+export interface PausedWorkflowRunRow {
+  runId: string; taskId: number; taskTitle: string | null; coordinator: string; callerAgentId: string | null;
+  teamId: string | null; owner: string | null; pause: string;
+}
+
+let pauseColumnOn: unknown;
+/** The column appears when the workflow module first registers after #3757; until then nothing is paused. */
+function pauseColumnReady(): boolean {
+  const db = getDb();
+  if (pauseColumnOn === db) return true;
+  const ready = (db.query(`PRAGMA table_info(${WORKFLOW_RUN_TABLE})`).all() as Array<{ name: string }>).some((column) => column.name === "pause");
+  if (ready) pauseColumnOn = db;
+  return ready;
+}
+
+export function pausedWorkflowRuns(): PausedWorkflowRunRow[] {
+  if (!workflowTablesReady() || !pauseColumnReady()) return [];
+  return getDb().query(`SELECT run.id AS runId, run.taskId AS taskId, task.title AS taskTitle, run.coordinator AS coordinator, run.callerAgentId AS callerAgentId,
+      task.team_id AS teamId, task.owner AS owner, run.pause AS pause
+    FROM ${WORKFLOW_RUN_TABLE} run LEFT JOIN tasks task ON task.id = run.taskId
+    WHERE run.scope = ? AND run.status = 'running' AND run.pause IS NOT NULL ORDER BY run.id`).all(WORKFLOW_SCOPE) as PausedWorkflowRunRow[];
+}
+
+/** #3757 — a paused run keeps its idle step agents: the idle reapers leave them alone until it is unpaused. */
+export function workflowRunPaused(runId: string): boolean {
+  if (!workflowTablesReady() || !pauseColumnReady()) return false;
+  return Boolean(getDb().query(`SELECT 1 FROM ${WORKFLOW_RUN_TABLE} WHERE scope = ? AND id = ? AND status = 'running' AND pause IS NOT NULL`).get(WORKFLOW_SCOPE, runId));
+}

package/src/db/write-ops.ts

@@ -1,3 +1,4 @@
+import { INTEGRATIONS_WRITE_OPS } from "./integrations-ops.ts";
 import { bufferedLastSeen, type PresenceSnapshot } from "./presence-buffer";
 import { ensureDurablePayloadSchema, readRetryReceipt, storeDurablePayload } from "./durable-payloads";
 import { ledgerSlotPredicate, ledgerSlots, WRITER_IDEMPOTENCY_RETENTION_MS } from "./writer-ledger";
@@ -5,6 +6,7 @@ import { workspaceChangeListeners } from "./workspaces";
 import { withWriterOpState } from "./main-thread-state";
 import { importVaultRows } from "./secrets-vault-import";
 import { PROMPT_VERSION_WRITE_OPS } from "./prompt-versions-ops";
+import { PROJECT_PROFILE_INSTRUCTION_WRITE_OPS } from "./project-profile-instructions";
 import { VAULT_WRITE_OPS } from "./secrets-vault-write";
 import { VAULT_EXPIRY_WRITE_OPS } from "./vault-expiry";
 import { VAULT_BACKUP_WRITE_OPS } from "./vault-backup";
@@ -117,6 +119,7 @@ import { RESTART_EPOCH_WRITE_OPS } from "./restart-epoch-write-ops.ts";
 import { TOKEN_USAGE_WRITE_OPS } from "./token-usage-write-ops.ts";
 // #1870 S3 — the 1 s raw writers: maintenance leases/runs, pending issue closes, workspace lifecycle.
 import { MAINTENANCE_JOB_WRITE_OPS } from "./maintenance-job-write-ops.ts";
+import { ISSUE_REFS_WRITE_OPS } from "./issue-refs-write-ops.ts";
 import { LONG_RUNNING_WORK_WRITE_OPS } from "./long-running-work-write-ops.ts";
 import { WORKSPACE_WRITE_OPS } from "./workspace-write-ops.ts";
 
@@ -406,6 +409,7 @@ export function allWriterOps(): Readonly<Record<string, WriterOpSpec>> {
   ...ROUTING_POLICY_WRITE_OPS,
       ...USER_SETTINGS_WRITE_OPS,
   ...PROMPT_VERSION_WRITE_OPS,
+  ...PROJECT_PROFILE_INSTRUCTION_WRITE_OPS,
       ...TOKEN_DB_WRITE_OPS,
       ...DELIVERY_WRITE_OPS,
       ...TIMER_WRITE_OPS,
@@ -440,6 +444,7 @@ export function allWriterOps(): Readonly<Record<string, WriterOpSpec>> {
     ...SPAWN_TREE_CLEANUP_WRITE_OPS,
     ...PROJECTS_WRITE_OPS,
     ...CHANNELS_WRITE_OPS,
+    ...INTEGRATIONS_WRITE_OPS,
     ...CI_INCIDENTS_WRITE_OPS,
     ...TEAMS_WRITE_OPS,
     ...TEAM_SPAWN_DELEGATION_WRITE_OPS,
@@ -468,6 +473,7 @@ export function allWriterOps(): Readonly<Record<string, WriterOpSpec>> {
     ...TOKEN_USAGE_WRITE_OPS,
     ...MAINTENANCE_JOB_WRITE_OPS,
     ...LONG_RUNNING_WORK_WRITE_OPS,
+    ...ISSUE_REFS_WRITE_OPS,
     ...WORKSPACE_WRITE_OPS,
     ...WEB_TOOL_CALL_WRITE_OPS,
     ...CLAUDE_API_BUDGET_WRITE_OPS,

package/src/db/writer-gateway.ts

@@ -174,8 +174,9 @@ export interface WriterStats {
    * SQLite lock (H4 §3.6.1). A wedged writer is NOT idle — this is deliberately distinct from
    * `busy:0`, which is how the naive pool-replace path mis-reported the stall. New non-critical
    * writes shed and critical writes are held until the wedged op completes and the FIFO resumes.
+   * #3776 — names the op holding the writer and how long it has run, so health says WHICH job; null when not wedged.
    */
-  wedged: boolean;
+  wedged: { op: string; runningMs: number } | null;
   /**
    * #2276 — how many times the IN-PROCESS drain has yielded the event loop mid-convoy because its
    * time slice was spent. 0 on the worker backend (nothing to slice). A steadily climbing count is
@@ -259,7 +260,7 @@ let drainYields = 0;
 // lock (§3.6.1). Cleared when that in-flight job finally settles (the op completed and released the
 // lock), which re-kicks the FIFO drain. Worker backend only — an in-process op runs synchronously on
 // the main thread, so it cannot be observed mid-stall.
-let wedged = false;
+let wedgedOp: { op: string; startedAt: number } | undefined;
 
 // Test-only fault injection for the precise H5 window: the op has COMMITTED, but its
 // acknowledgement is lost before the caller observes the result. One-shot by design so a retry
@@ -355,14 +356,14 @@ function createFileWriter(dbPath: string): WorkerBackend {
 
 /** Server shutdown keeps already-admitted writes; test/reset teardown remains immediate. */
 export async function drainWriterOnShutdown(): Promise<void> {
-  try { await shutdownDrain.start(() => !draining && !wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0); }
+  try { await shutdownDrain.start(() => !draining && !wedgedOp && pendingWriteDepth() === 0 && inFlightWrites() === 0); }
   finally { await shutdownWriter(); }
 }
 
 export async function shutdownWriter(): Promise<void> {
   const current = backend;
   backend = undefined;
-  wedged = false;
+  wedgedOp = undefined;
   // Reject anything still queued at the gateway so no caller hangs across a teardown.
   const stranded = submissionQueue.splice(0, submissionQueue.length);
   for (const entry of stranded) {
@@ -379,7 +380,8 @@ export function isWriterInitialized(): boolean {
 }
 
 export function writerStats(): WriterStats {
-  if (!backend) return { backend: "uninitialized", queued: 0, busy: 0, highWater: queueHighWater, wedged: false, drainYields };
+  if (!backend) return { backend: "uninitialized", queued: 0, busy: 0, highWater: queueHighWater, wedged: null, drainYields };
+  const wedged = wedgedOp ? { op: wedgedOp.op, runningMs: Math.round(performance.now() - wedgedOp.startedAt) } : null;
   if (backend.kind === "in-process") {
     return { backend: "in-process", queued: pendingWriteDepth(), busy: inFlightWrites(), highWater: queueHighWater, wedged, drainYields };
   }
@@ -480,7 +482,7 @@ function admitOrThrow(op: string, opts: WriteOpts | undefined, count: number): v
   const occupancy = pendingWriteDepth() + inFlightWrites();
   if (occupancy + count > queueHardCap) throw new WriterOverloadedError(op, occupancy);
   if (opts?.critical) return;
-  const workerWedged = wedged;
+  const workerWedged = wedgedOp !== undefined;
   if (workerWedged || occupancy + count > queueHighWater) throw new WriterOverloadedError(op, occupancy);
 }
 
@@ -511,7 +513,7 @@ async function drainLoop(): Promise<void> {
     // second concurrent drain and anything submitted meanwhile is picked up by this same loop when
     // it resumes — FIFO is preserved, the queue is only PAUSED.
     let sliceStartedAt = performance.now();
-    while (submissionQueue.length > 0 && !wedged) {
+    while (submissionQueue.length > 0 && !wedgedOp) {
       const entry = submissionQueue.shift()!;
       const onLoop = backend?.kind === "in-process";
       await dispatchEntry(entry);
@@ -521,7 +523,7 @@ async function drainLoop(): Promise<void> {
         sliceStartedAt = performance.now();
         continue;
       }
-      if (submissionQueue.length === 0 || wedged) continue;
+      if (submissionQueue.length === 0 || wedgedOp) continue;
       if (performance.now() - sliceStartedAt < drainSliceMs) continue;
       drainYields += 1;
       await yieldEventLoop();
@@ -529,7 +531,7 @@ async function drainLoop(): Promise<void> {
     }
   } finally {
     draining = false;
-    shutdownDrain.notify(!wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0);
+    shutdownDrain.notify(!wedgedOp && pendingWriteDepth() === 0 && inFlightWrites() === 0);
   }
 }
 
@@ -650,6 +652,7 @@ async function dispatchWorker(entry: Submission, back: WorkerBackend): Promise<v
 
   // retrySafe stays false (default): a write must NEVER be auto-re-dispatched (it would repeat its
   // effect). No pool timeout is armed — the deadline is the gateway's own (§3.6.1).
+  const startedAt = performance.now();
   const poolPromise = back.pool.run<WriterJobInput, WriterJobResult>(WRITER_TASK_NAME, input, { labels });
 
   let timer: ReturnType<typeof setTimeout> | undefined;
@@ -665,13 +668,16 @@ async function dispatchWorker(entry: Submission, back: WorkerBackend): Promise<v
     // WEDGED (§3.6.1). The op is still running and still holds the lock; do NOT replace the worker.
     // Surface the caller's INDETERMINATE outcome, hold the FIFO, and resume when the op finally
     // settles (it commits/rolls back and releases the lock on the SAME warm connection).
-    wedged = true;
+    // #3776 — name the op: "writer wedged" alone left an incident to guess which job held the lock.
+    wedgedOp = { op: entry.op, startedAt };
+    console.warn(`[writer-gateway] writer op ${entry.op} has held the writer past its ${writerTimeoutMs}ms deadline; writes are held until it settles`);
     rejectIndeterminate(entry);
     poolPromise.then(
       () => {},
       () => {},
     ).finally(() => {
-      wedged = false;
+      console.warn(`[writer-gateway] wedged writer op ${entry.op} settled after ${Math.round(performance.now() - startedAt)}ms`);
+      wedgedOp = undefined;
       kickDrain();
     }).catch((error) => console.error(`[writer-gateway] resume after wedge failed: ${errMessage(error)}`));
     return;

package/src/dispatch-availability.ts

@@ -1,5 +1,5 @@
-import type { HostView, Orchestrator } from "@humwick/sdk";
-import type { ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
+import { errMessage, type HostView, type Orchestrator } from "@humwick/sdk";
+import { resolveProviderSelection, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
 import type { DispatchTarget } from "./task-dispatcher";
 import { spawnPlacementRefusal, hostSpawnAdmission, hostIdentitiesById, spawnHostCapAdmission, type SpawnCapOptions } from "./spawn-placement";
 import { liveHostDisabledProviders } from "./host-provider-store";
@@ -17,13 +17,15 @@ export interface LiveTargetAdmission {
   targetUnavailableReason: (target: DispatchTarget) => string | undefined;
 }
 
-export function liveTargetAdmission(orchestrators: readonly Orchestrator[], views: readonly HostView[], orchestratorId?: string, capOpts: SpawnCapOptions = {}): LiveTargetAdmission {
+export function liveTargetAdmission(orchestrators: readonly Orchestrator[], views: readonly HostView[], catalog: ProviderCatalogEntry[], orchestratorId?: string, capOpts: SpawnCapOptions = {}): LiveTargetAdmission {
   const identities = hostIdentitiesById(views);
   const byId = new Map(views.map((view) => [view.id, view]));
   const disabled = liveHostDisabledProviders();
   const capBlock = spawnHostCapAdmission(capOpts);
   const hosts = orchestrators.filter((host) => !orchestratorId || host.id === orchestratorId);
   const targetAdmission = ({ provider, model, effort }: DispatchTarget): TargetAdmission => {
+    try { resolveProviderSelection({ provider, model, effort }, catalog); }
+    catch { return { eligible: false, uncertainties: [] }; }
     let eligible = false;
     const uncertainties: TargetUncertainty[] = [];
     for (const host of hosts) {
@@ -34,9 +36,11 @@ export function liveTargetAdmission(orchestrators: readonly Orchestrator[], view
     }
     return { eligible, uncertainties };
   };
-  const targetUnavailableReason = (target: DispatchTarget) => spawnPlacementRefusal(
-    orchestrators, views, target.provider, target.model, target.effort, orchestratorId, capOpts,
-  );
+  const targetUnavailableReason = (target: DispatchTarget) => {
+    try { resolveProviderSelection(target, catalog); }
+    catch (error) { return errMessage(error); }
+    return spawnPlacementRefusal(orchestrators, views, target.provider, target.model, target.effort, orchestratorId, capOpts);
+  };
   return { targetAdmission, targetUnavailableReason };
 }
 

package/src/ephemeral/{landing-ingress-proof-worker.ts → landing-ingress-proof.test-fixture.ts}

@@ -1,7 +1,7 @@
 /** Precondition setup for the packaged-ephemeral landing-ingress Proof of Life.
  * Delivery and both crash boundaries run through the packaged Relay; this fixture only registers
  * the disposable workspace that its real ephemeral orchestrator will probe. */
-import { initDb } from "../db/schema.ts";
+import { closeDb, openExistingDb } from "../db/connection.ts";
 import { upsertWorkspace } from "../db/workspaces.ts";
 
 const dbPath = process.env.DB_PATH;
@@ -13,7 +13,8 @@ if (!dbPath || !worktreePath || !orchestratorId || !baseSha) {
   throw new Error("usage: landing-ingress-proof-worker <worktree-path> <orchestrator-id> <base-sha>");
 }
 
-initDb(dbPath);
+// The installed Relay already owns the schema; a fixture must not migrate its live DB (#3715).
+openExistingDb(dbPath);
 upsertWorkspace({
   id: workspaceId,
   repoRoot: worktreePath,
@@ -27,4 +28,5 @@ upsertWorkspace({
   orchestratorId,
   metadata: {},
 });
+closeDb();
 console.log(JSON.stringify({ workspaceId }));

package/src/host/disk-alert.ts

@@ -0,0 +1,121 @@
+// #3762 — warn before a host disk fills. Pure: the single-writer utilization op settles each beat
+// against the stored one (same seam as gate-capacity-alert.ts), so each level change alerts once.
+// Two triggers, one level ladder: used% crossing 85/92/97, or the free-space history projecting
+// full within 24 h (6 h = high). An episode alerts once per worse level and clears with a notice.
+import type { HostDiskWatch, HostMachineSample } from "@humwick/sdk";
+
+export type DiskLevel = 1 | 2 | 3;
+
+/** Used-share rungs: warn / high / critical. */
+export const DISK_USED_PCT_LEVELS: readonly [number, number, number] = [85, 92, 97];
+/** An episode only clears below this, so a disk hovering on a rung does not flap. */
+export const DISK_CLEAR_USED_PCT = 82;
+export const DISK_ETA_WARN_H = 24;
+export const DISK_ETA_HIGH_H = 6;
+/** Growth needs this much history before it is a rate rather than a blip. */
+export const DISK_MIN_TREND_SPAN_MS = 3 * 3_600_000;
+const HISTORY_SPACING_MS = 30 * 60_000;
+const HISTORY_MAX_SPAN_MS = 25 * 3_600_000;
+const TREND_WINDOW_MS = 24 * 3_600_000;
+
+export type DiskAlert =
+  | {
+    kind: "low"; level: DiskLevel; since: number; freeMb: number; totalMb: number; path?: string;
+    /** Free-space decline over the trend window, MB/h; present only while declining. */
+    rateMbPerHour?: number; etaHours?: number; spanHours?: number;
+  }
+  | { kind: "recovered"; since: number; freeMb: number; totalMb: number; path?: string };
+
+export interface DiskNotice {
+  subject: string;
+  body: string;
+  priority: "low" | "default" | "high" | "urgent";
+  idempotencyKey: string;
+}
+
+function usageLevel(usedPct: number): number {
+  return DISK_USED_PCT_LEVELS.filter((rung) => usedPct >= rung).length;
+}
+
+function growthLevel(etaHours: number | undefined): number {
+  if (etaHours === undefined) return 0;
+  return etaHours < DISK_ETA_HIGH_H ? 2 : etaHours < DISK_ETA_WARN_H ? 1 : 0;
+}
+
+function nextHistory(prev: HostDiskWatch["history"], at: number, freeMb: number): Array<[number, number]> {
+  const kept = (prev ?? []).filter(([t]) => t >= at - HISTORY_MAX_SPAN_MS && t < at);
+  const last = kept[kept.length - 1];
+  if (!last || at - last[0] >= HISTORY_SPACING_MS) kept.push([at, freeMb]);
+  return kept;
+}
+
+/** Decline rate over the oldest point inside the trend window, or undefined (too little history / not declining). */
+function trend(history: Array<[number, number]>, at: number, freeMb: number): { rateMbPerHour: number; etaHours: number; spanHours: number } | undefined {
+  const oldest = history.find(([t]) => t >= at - TREND_WINDOW_MS);
+  if (!oldest || at - oldest[0] < DISK_MIN_TREND_SPAN_MS) return undefined;
+  const declineMb = oldest[1] - freeMb;
+  if (declineMb <= 0) return undefined;
+  const spanHours = (at - oldest[0]) / 3_600_000;
+  const rateMbPerHour = declineMb / spanHours;
+  return { rateMbPerHour, etaHours: freeMb / rateMbPerHour, spanHours };
+}
+
+/**
+ * Stamps the relay-owned disk watch onto the stored sample and names the one alert owed, if any.
+ * A beat with no disk pair keeps the previous watch (missing telemetry is not a state change).
+ */
+export function settleDiskWatch(prev: HostMachineSample | undefined, next: HostMachineSample): { machine: HostMachineSample; alert?: DiskAlert } {
+  const before = prev?.diskWatch;
+  const { diskFreeMb: freeMb, diskTotalMb: totalMb } = next;
+  if (freeMb === undefined || totalMb === undefined || totalMb <= 0) {
+    return { machine: before ? { ...next, diskWatch: before } : next };
+  }
+  const history = nextHistory(before?.history, next.at, freeMb);
+  const projected = trend(history, next.at, freeMb);
+  const usedPct = ((totalMb - freeMb) / totalMb) * 100;
+  const level = Math.max(usageLevel(usedPct), growthLevel(projected?.etaHours));
+  const prevLevel = before?.level ?? 0;
+  const where = next.diskPath ? { path: next.diskPath } : {};
+
+  if (level > prevLevel) {
+    const since = before?.since ?? next.at;
+    const watch: HostDiskWatch = { level: level as DiskLevel, since, history };
+    return {
+      machine: { ...next, diskWatch: watch },
+      alert: {
+        kind: "low", level: level as DiskLevel, since, freeMb, totalMb, ...where,
+        ...(projected ? { rateMbPerHour: projected.rateMbPerHour, etaHours: projected.etaHours, spanHours: projected.spanHours } : {}),
+      },
+    };
+  }
+  if (prevLevel > 0 && level === 0 && usedPct < DISK_CLEAR_USED_PCT) {
+    return { machine: { ...next, diskWatch: { history } }, alert: { kind: "recovered", since: before!.since ?? next.at, freeMb, totalMb, ...where } };
+  }
+  return { machine: { ...next, diskWatch: { ...(before?.level ? { level: before.level, since: before.since } : {}), history } } };
+}
+
+const gb = (mb: number) => `${(mb / 1024).toFixed(mb >= 10_240 ? 0 : 1)} GB`;
+
+export function composeDiskNotice(hostId: string, label: string, alert: DiskAlert): DiskNotice {
+  const volume = alert.path ? ` (${alert.path})` : "";
+  const usedPct = Math.round(((alert.totalMb - alert.freeMb) / alert.totalMb) * 100);
+  if (alert.kind === "recovered") {
+    return {
+      subject: `Disk recovered on ${label}`,
+      body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used); the low-disk episode is over. (#3762)`,
+      priority: "low",
+      idempotencyKey: `host-disk:${hostId}:${alert.since}:recovered`,
+    };
+  }
+  const word = alert.level === 3 ? "CRITICAL" : alert.level === 2 ? "very low" : "low";
+  const growth = alert.rateMbPerHour !== undefined && alert.etaHours !== undefined
+    ? ` Free space fell ~${gb(alert.rateMbPerHour * 24)}/day over the last ${Math.round(alert.spanHours ?? 0)} h; at that rate it is full in ~${Math.max(1, Math.round(alert.etaHours))} h.`
+    : "";
+  return {
+    subject: `Disk ${word} on ${label}: ${gb(alert.freeMb)} free (${usedPct}% used)`,
+    body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used).${growth} `
+      + "Check workspaces, runtime logs and caches on that host before agents start failing with ENOSPC. (#3762)",
+    priority: alert.level === 3 ? "urgent" : alert.level === 2 ? "high" : "default",
+    idempotencyKey: `host-disk:${hostId}:${alert.since}:${alert.level}`,
+  };
+}

package/src/lifecycle-manager.ts

@@ -53,7 +53,8 @@ const DEFAULT_COMPACT_TARGET = 0.3;
 const DEFAULT_MAX_TASKS_BEFORE_COMPACT = 10;
 const DEFAULT_COMPACT_COOLDOWN_MS = 60_000;
 
-interface LifecycleManagerOptions { tickMs?: number; now?: () => number; buildManagedSpawnParams?: typeof buildManagedSpawnParamsAsync; }
+type ManagedSpawnParamsBuilder = (...args: Parameters<typeof buildManagedSpawnParamsAsync>) => Record<string, unknown> | Promise<Record<string, unknown>>;
+interface LifecycleManagerOptions { tickMs?: number; now?: () => number; buildManagedSpawnParams?: ManagedSpawnParamsBuilder; }
 
 // #1021 A1 — the outcome of trying to wake an on-demand policy via a queued message. A
 // caller (e.g. wakeRepoSteward) must be able to tell "a live agent will handle this" from
@@ -66,7 +67,7 @@ export class LifecycleManager {
   private timer?: Timer;
   private readonly tickMs: number;
   private readonly now: () => number;
-  private readonly buildManagedSpawnParams: typeof buildManagedSpawnParamsAsync; private readonly pendingSpawnPreparations = new Map<string, Promise<Command | null>>();
+  private readonly buildManagedSpawnParams: ManagedSpawnParamsBuilder; private readonly pendingSpawnPreparations = new Map<string, Promise<Command | null>>();
   private readonly updateRestartPending = new Set<string>();
   // #1761 — how many times a policy's stop has been forcefully re-issued while the process refused to
   // confirm dead. Bounds the escalation so a wedged steward is retried, then given up to `failed`,

package/src/maintenance/cd-watcher.ts

@@ -140,7 +140,7 @@ export async function runCdWatcherJob(context: MaintenanceJobContext = { signal:
   // `offThread: true` mirrors the #1725 off-thread jobs (old-message-prune, db-maintenance, …): it
   // records in the run's persisted result that the heavy work ran in a subprocess, so an operator
   // reading the last result sees the same proof the run registry acted on for breaker attribution.
-  return { ...result, offThread: true, transition: notifyMainCiTransition(result.observation) };
+  return { ...result, offThread: true, transition: await notifyMainCiTransition(result.observation) };
 }
 
 /** Best-effort Cosmos deploy hero-shot: read the ledger row this deploy just
@@ -164,9 +164,9 @@ async function emitDeployHeroShot(sha: string | null): Promise<void> {
 }
 
 /** Best-effort transition alert; never throws into the deploy job. */
-function notifyMainCiTransition(observation: CiObservation): Record<string, unknown> {
+async function notifyMainCiTransition(observation: CiObservation): Promise<Record<string, unknown>> {
   try {
-    const { transition, incidentId, notified, messages } = notifyCiTransition(observation);
+    const { transition, incidentId, notified, messages } = await notifyCiTransition(observation);
     return { flip: transition, notified, messages, ...(incidentId ? { incidentId } : {}) };
   } catch (error) {
     const message = errMessage(error);

package/src/maintenance/critical-job-watchdog.ts

@@ -17,12 +17,13 @@
 //   - the configured fleet-ops agent target, so something that can actually act is woken.
 
 import { errMessage } from "@humwick/sdk";
-import { getConfig, setConfig } from "../config-store";
+import { getConfig } from "../config-store";
+import { setConfigAsync } from "../config-store-write";
+import { resetExpiredCriticalRunsAsync } from "../db/maintenance-job-runs-async";
 import { stewardFallbackTarget } from "../config";
 import {
   evaluateCriticalMaintenanceJobs,
   legacyHungCriticalMaintenanceRuns,
-  resetExpiredCriticalMaintenanceRuns,
   type CriticalJobCondition,
   type LegacyHungCriticalMaintenanceRun,
   type ResetCriticalMaintenanceRun,
@@ -69,11 +70,11 @@ export interface CriticalJobWatchdogResult {
   suppressed?: "unchanged-within-reminder-window" | "awaiting-second-failure";
 }
 
-export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWatchdogResult {
+export async function runCriticalJobWatchdog(now: number = Date.now()): Promise<CriticalJobWatchdogResult> {
   // Do this before evaluating health so the durable scheduler latch is recovered even if the
   // notification path is unavailable.  A normal timed-out run clears itself; this is only the
   // safety net for a row stranded in `running` after its own deadline lease elapsed.
-  const resetRuns = resetExpiredCriticalMaintenanceRuns(now);
+  const resetRuns = await resetExpiredCriticalRunsAsync(now);
   const legacyHungRuns = legacyHungCriticalMaintenanceRuns(now);
   const conditions = evaluateCriticalMaintenanceJobs(now);
   const errors = conditions.filter((condition) => condition.status === "error");
@@ -86,9 +87,9 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     state.missedChecks[key] = checks;
     return checks >= 2;
   });
-  const persist = () => {
+  const persist = async () => {
     if (JSON.stringify(state) !== JSON.stringify(prior)) {
-      setConfig(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
+      await setConfigAsync(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
     }
   };
   const heldRestartEpoch = currentHeldRestartEpoch(now);
@@ -97,7 +98,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     // Recovered (or never degraded): clear the marker so the NEXT degradation notifies immediately
     // rather than being swallowed by the reminder window opened by the previous one.
     state.lastNotified = null;
-    persist();
+    await persist();
     return errors.length > 0
       ? { status: "error", degradedJobIds: errors.map((condition) => condition.jobId), notified: false, recipients: [], suppressed: "awaiting-second-failure" }
       : { status: conditions.length > 0 ? "warn" : "ok", degradedJobIds: [], notified: false, recipients: [] };
@@ -115,7 +116,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     ...(heldRestartEpoch ? [`epoch:${heldRestartEpoch.id}:${heldRestartEpoch.blockerCode}`] : []),
   ].sort().join("|");
   if (state.lastNotified?.fingerprint === fingerprint && now - state.lastNotified.notifiedAt < DEGRADED_REMINDER_MS) {
-    persist();
+    await persist();
     return {
       status: "error", degradedJobIds,
       ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -154,7 +155,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
   }
 
   state.lastNotified = { fingerprint, notifiedAt: now };
-  persist();
+  await persist();
   return {
     status: "error", degradedJobIds,
     ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -236,16 +237,14 @@ function composeNotice(
 
 /** Handler wrapper: a watchdog that throws must not mark itself failed-and-silent, so failures are
  *  reported as a result the job record carries. */
-export function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Record<string, unknown> {
-  // #1819 — SYNCHRONOUS and single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify
-  // fan-out that must not be split (the `lastNotified` fingerprint is stamped only after the sends,
-  // so a half-run would re-page on the next tick). The only safe checkpoint is before it starts, and
-  // on one thread that is also the only one that could ever fire. Its work is bounded by the number
-  // of critical jobs (one today), so overrun is not a live risk; if that ever changes this is an
-  // #1809 offload candidate, not a place for more checks.
+export async function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  // #1819 — single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify fan-out that must not
+  // be split (the `lastNotified` fingerprint is stamped only after the sends, so a half-run would re-page
+  // on the next tick). The only safe checkpoint is before it starts. Its two writes (the lease reset and
+  // the state persist) are writer ops (#1870 S7).
   if (context.signal?.aborted) return { status: "ok", aborted: true, degradedJobIds: [], notified: false, recipients: [] };
   try {
-    return { ...runCriticalJobWatchdog() };
+    return { ...(await runCriticalJobWatchdog()) };
   } catch (error) {
     return { status: "error", error: errMessage(error) };
   }

package/src/maintenance/index.ts

@@ -1,6 +1,7 @@
 export {
   cancelMaintenanceRun,
   listMaintenanceJobs,
+  listMaintenanceJobsAsync,
   listRecentMaintenanceRuns,
   listRunningMaintenanceRuns,
   resetOrphanTrackerForTests,