@alphacoder-v0/pi-loops 0.17.3

package/src/trigger-runtime.ts

@@ -0,0 +1,711 @@
+/**
+ * The trigger runtime: takes Trigger envelopes from sources (the periodic dynamic
+ * checker, MCP pushes), dedups them, runs the dynamic-rule sub-agent (or delivers a
+ * notification straight into the chat), marks fire-once rules, promotes results
+ * when asked, and keeps audit + running state for /triggers.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { createHash } from "node:crypto";
+import { realpathish } from "./paths.ts";
+import { type PresenceEntry, type PresenceSelf, chooseRuleOwner, isSelf, withinProject } from "./presence.ts";
+import { capRedacted, previewRedacted } from "./redact.ts";
+import { type RunnerResult, type SubagentRunner, failedRun } from "./runner.ts";
+import { type SubagentSlot, SubagentSlots } from "./slots.ts";
+import type { JobStore } from "./store.ts";
+import { backoffWaitMs, stamp } from "./schedule.ts";
+import {
+	DEFAULT_TRIGGER_POLL_INTERVAL_SECS,
+	DedupWindow,
+	NO_MATCH_SENTINEL,
+	PollLedger,
+	SUMMARY_CAP_CHARS,
+	type DynamicTriggerRule,
+	type Trigger,
+	TriggerStore,
+	buildPeriodicCheckTrigger,
+	extractDynamicRuleIds,
+	renderDynamicTriggerPrompt,
+ newTraceId } from "./triggers.ts";
+
+export type TriggerDelivery = "sub_agent" | "inject_summary" | "inject_and_run";
+
+export interface RunningTrigger {
+	traceId: string;
+	sourceLabel: string;
+	eventLabel: string;
+	startedAt: string;
+	promptPreview: string;
+	cwd: string;
+	ctrl: AbortController;
+}
+
+export interface TriggerOutcome {
+	trigger: Trigger;
+	delivery: TriggerDelivery;
+	ok: boolean;
+	matchedRules: DynamicTriggerRule[];
+	summary: string;
+	error?: string;
+	durationMs: number;
+	cost: number;
+	promoted: boolean;
+	sessionFile?: string;
+}
+
+/** Where a promotion / injection actually went: the chat of this process, or the inbox (wrong project here). */
+export type PromoteTarget = "chat" | "inbox";
+
+export interface TriggerRuntimeHooks {
+	/** Insert text into the parent chat context (no model turn). Already carries the `[Trigger …]` prefix. Returns where it went. */
+	onPromote?: (content: string, trigger: Trigger) => PromoteTarget | Promise<PromoteTarget>;
+	/** Inject `prompt` into the parent chat and run one model turn. Returns where it went. */
+	onInjectAndRun?: (prompt: string, trigger: Trigger) => PromoteTarget | Promise<PromoteTarget>;
+	onStarted?: (running: RunningTrigger) => void;
+	onFinished?: (outcome: TriggerOutcome) => void;
+	log?: (message: string) => void;
+}
+
+export interface TriggerRuntimeOptions {
+	store: TriggerStore;
+	jobStore: JobStore;
+	getSession: () => { sessionId?: string; cwd: string; model?: string; thinking?: string; trusted?: boolean };
+	hooks?: TriggerRuntimeHooks;
+	pollIntervalSecs?: number;
+	/** Runs check/action sub-agents (in-process through pi's SDK in production; tests inject a fake). */
+	runner: SubagentRunner;
+	now?: () => number;
+	/** Shared dedup file (`<dir>/dedup.json`); omit for an in-memory window. */
+	dedupFile?: string;
+	/** Trigger hop of this process (0 for the interactive pi); children get hop + 1. */
+	hop?: number;
+	/**
+	 * Presence of every pi on the machine and this process's own identity. With both, a project's
+	 * checks and push evaluations are run by a pi open in that project (preferring the session
+	 * that created the rules); without them the machine leader does everything (tests).
+	 */
+	presence?: () => PresenceEntry[];
+	self?: PresenceSelf;
+	/** Whether this process is the machine leader right now (used outside tick(), e.g. for pushes). */
+	isLeader?: () => boolean;
+	/** Default cap on a check/action sub-agent (`[triggers] run_timeout_secs`); a rule's `timeoutMs` overrides it. */
+	runTimeoutMs?: number;
+	/** How long a push deferred to another process waits for that process to claim it (see `admit`). */
+	deferredTakeoverMs?: number;
+	/**
+	 * Checks and actions in flight at once. Spawning every accepted trigger concurrently is
+	 * bounded in practice by a person watching the feed; a headless host has nobody watching, and a
+	 * server pushing distinct events would otherwise open one sub-agent per event. Only used to size
+	 * a private counter when `slots` is absent.
+	 */
+	maxConcurrent?: number;
+	/**
+	 * The process-wide sub-agent admission counter, shared with the scheduler and the /goal evaluator
+	 * (src/slots.ts). Omitted, the trigger runtime bounds only its own checks against `maxConcurrent`
+	 * — which is the bug this exists to close, so production always passes one.
+	 */
+	slots?: SubagentSlots;
+	/** Today's automation spend against `[limits] daily_budget_usd`, shared with the scheduler. */
+	budget?: () => { spent: number; cap: number; over: boolean };
+}
+
+export const DEFAULT_MAX_CONCURRENT_CHECKS = 3;
+
+/**
+ * Pushes held while every slot is busy. Retries drain at most `maxConcurrent` (3) per scheduler
+ * tick (30s), so 32 is about five minutes of backlog — the width of the dedup window that defines
+ * a push's identity. Past that the held events are older than the window they would collapse in,
+ * and holding more of them only delays the moment we admit the machine cannot keep up: the oldest
+ * is dropped, with an audit row, instead of the list growing until the process dies.
+ */
+export const MAX_PENDING_PUSHES = 32;
+
+/**
+ * Backoff for a rule whose check keeps failing — the same shape and numbers as the scheduler's job
+ * backoff (`FAILURE_BACKOFF_*` in scheduler.ts), which exists for exactly this: a check that failed
+ * three times in a row is failing for a reason the next poll will not fix, and each attempt bills.
+ */
+export const CHECK_FAILURE_BACKOFF_AFTER = 3;
+export const CHECK_FAILURE_BACKOFF_BASE_MS = 5 * 60_000;
+export const CHECK_FAILURE_BACKOFF_MAX_MS = 6 * 60 * 60_000;
+
+/**
+ * Transcripts are kept per project, not in one directory for the machine: a shared budget meant
+ * three projects polling every ten minutes exhausted it within hours, taking the evidence for
+ * "why did this rule not match?" with them.
+ *
+ * The basename alone was not per project: `~/a/web` and `~/b/web` landed in one directory, back to
+ * one shared budget of 40 — and each project's prune deleted the other's evidence. So the readable
+ * basename keeps its place and a short digest of the resolved path makes it unique. Resolved the way
+ * everything else here compares paths (`realpathish`), so a worktree reached through a symlink is
+ * still the same project.
+ */
+export function triggerSessionKey(cwd: string): string {
+	const base = path.basename(cwd || "unknown").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown";
+	const digest = createHash("sha256").update(realpathish(cwd || "unknown")).digest("hex").slice(0, 8);
+	return `triggers-${base}-${digest}`;
+}
+
+/** Check transcripts kept per project, as docs/configuration.md promises. */
+export const TRIGGER_SESSIONS_KEPT = 40;
+
+export const DEFAULT_TRIGGER_RUN_TIMEOUT_MS = 15 * 60_000;
+/**
+ * A push handed to the presence-chosen owner is only really handled if that process has the MCP
+ * server connected — it authenticates per process, so it may not. The receiving process waits this
+ * long for the owner to claim the machine-wide dedup key and takes the push back if it does not.
+ */
+export const DEFAULT_DEFERRED_TAKEOVER_MS = 5_000;
+
+/** The engine prefixes `[Trigger <trace>] ` and injects the payload summary itself, capped. */
+export function promotionBody(trigger: Trigger, summary: string): string {
+	// Capped, never reflowed: a promoted result is often a file, a diff or test output, and rewrapping
+	// it is how a diff stops applying and a stack trace stops pointing anywhere. Truncation respects
+	// character boundaries, so the cap cannot cut a multi-byte character in half.
+	return `[Trigger ${trigger.traceId}] ${capRedacted(summary, SUMMARY_CAP_CHARS)}`;
+}
+
+/**
+ * A sub-agent result promoted into the chat: `<source> fired <event>.\nResult: …`, so the chat says
+ * what fired and not only what came back — a result with no cause reads as the agent talking to
+ * itself. The rendered body is then capped at `SUMMARY_CAP_CHARS` characters — the same cap an audit
+ * row's summary gets, and characters rather than bytes, so it cannot cut a multi-byte one in half.
+ */
+export function promotionSummaryBody(trigger: Trigger, summary: string): string {
+	return capRedacted(`[Trigger ${trigger.traceId}] ${trigger.sourceLabel} fired ${trigger.eventLabel}.\nResult: ${summary}`, SUMMARY_CAP_CHARS);
+}
+
+/**
+ * The envelope fields carried on every audit row, in every state a trigger passes through. Without
+ * the idempotency key on each of them, "which pushes collapsed into which" cannot be answered after
+ * the fact — and that question is the whole reason the dedup window is allowed to drop anything.
+ * Bounded and redacted: a key is caller-supplied text.
+ */
+function envelopeOf(trigger: Trigger): Record<string, unknown> {
+	return { idempotency_key: capRedacted(trigger.idempotencyKey, 256), replacement_policy: trigger.replacementPolicy, received_at: trigger.receivedAt };
+}
+
+export class TriggerRuntime {
+	readonly store: TriggerStore;
+	private readonly jobStore: JobStore;
+	private readonly getSession: TriggerRuntimeOptions["getSession"];
+	private readonly hooks: TriggerRuntimeHooks;
+	private readonly runner: SubagentRunner;
+	private readonly now: () => number;
+	private readonly dedup: DedupWindow;
+	/** Pushes that inject into this window are deduplicated here only: every window reacts. */
+	private readonly localDedup = new DedupWindow();
+	private readonly ledger: PollLedger;
+	private readonly presence?: () => PresenceEntry[];
+	private readonly self?: PresenceSelf;
+	private readonly isLeader?: () => boolean;
+	private readonly hop: number;
+	/** Mutable so a config reload at session start takes effect. */
+	runTimeoutMs: number;
+	private readonly deferredTakeoverMs: number;
+	private readonly running = new Map<string, RunningTrigger>();
+	/** Pushes refused for want of a slot, waiting for the next tick to retry them (see `queuePush`). */
+	private readonly pending = new Map<string, { trigger: Trigger; delivery: TriggerDelivery }>();
+	/** Every sub-agent this process starts takes a slot from here, loop runs included. */
+	readonly slots: SubagentSlots;
+	private readonly budget: () => { spent: number; cap: number; over: boolean };
+	pollIntervalSecs: number;
+	lastCheckAt = 0;
+	/** Bounded, display-only status of the latest periodic check. */
+	lastPoll: { at: string; cwd: string; outcome: string; traceId: string; sourceLabel: string; eventLabel: string; summary: string } | undefined;
+	dedupedCount = 0;
+	cycleSuppressedCount = 0;
+
+	constructor(opts: TriggerRuntimeOptions) {
+		this.store = opts.store;
+		this.jobStore = opts.jobStore;
+		this.getSession = opts.getSession;
+		this.hooks = opts.hooks ?? {};
+		this.runner = opts.runner;
+		this.now = opts.now ?? Date.now;
+		this.pollIntervalSecs = opts.pollIntervalSecs ?? DEFAULT_TRIGGER_POLL_INTERVAL_SECS;
+		this.dedup = new DedupWindow(undefined, opts.dedupFile);
+		this.ledger = new PollLedger(opts.dedupFile ? path.join(path.dirname(opts.dedupFile), "polls.json") : undefined);
+		this.presence = opts.presence;
+		this.self = opts.self;
+		this.isLeader = opts.isLeader;
+		this.hop = opts.hop ?? 0;
+		this.runTimeoutMs = opts.runTimeoutMs ?? DEFAULT_TRIGGER_RUN_TIMEOUT_MS;
+		this.deferredTakeoverMs = opts.deferredTakeoverMs ?? DEFAULT_DEFERRED_TAKEOVER_MS;
+		this.slots = opts.slots ?? new SubagentSlots(opts.maxConcurrent ?? DEFAULT_MAX_CONCURRENT_CHECKS);
+		this.budget = opts.budget ?? (() => ({ spent: 0, cap: 0, over: false }));
+		// Audit writes are best-effort — a failed write never fails a trigger; say so once per distinct error.
+		let lastReported: string | undefined;
+		this.store.onPersistenceError ??= (message) => {
+			if (message === lastReported) return;
+			lastReported = message;
+			this.log(message);
+		};
+	}
+
+	/** The log hook is the UI's notifier in pi-loops; it must never turn into a rejection here. */
+	private log(message: string): void {
+		try {
+			this.hooks.log?.(message);
+		} catch {
+			/* nothing left to report to */
+		}
+	}
+
+	runningList(): RunningTrigger[] {
+		return [...this.running.values()];
+	}
+
+	/** Pushes waiting for a free slot: work that is late, not work that was lost. */
+	get pendingPushCount(): number {
+		return this.pending.size;
+	}
+
+	abort(traceId: string): boolean {
+		const r = this.running.get(traceId);
+		if (!r) return false;
+		r.ctrl.abort();
+		return true;
+	}
+
+	abortAll(): number {
+		const n = this.running.size;
+		for (const r of this.running.values()) r.ctrl.abort();
+		return n;
+	}
+
+	async stop(): Promise<void> {
+		this.abortAll();
+	}
+
+	/** The process that must evaluate `rule`; `undefined` when nobody owns it and the leader steps in. */
+	private ownerOf(rule: DynamicTriggerRule): PresenceEntry | undefined {
+		if (!this.self) return undefined;
+		return chooseRuleOwner(this.presence?.() ?? [], rule.cwd, this.self.host, rule.createdBy?.sessionId);
+	}
+
+	/**
+	 * Whether this process evaluates `rule`. A rule belongs to the session that created it, not to
+	 * its directory, so two pi windows in one repo each check their own rules and a result can only
+	 * be promoted into the chat that asked for it. `ownIfUnowned` covers only the rules nobody owns —
+	 * their creating session has closed and their project has no pi open — which the machine leader
+	 * checks, into the inbox. It never takes a rule away from the process that does own it.
+	 */
+	ownsRule(rule: DynamicTriggerRule, ownIfUnowned: boolean): boolean {
+		if (!this.self) return ownIfUnowned;
+		const owner = this.ownerOf(rule);
+		return owner ? isSelf(owner, this.self) : ownIfUnowned;
+	}
+
+	/**
+	 * The slot a rule's work occupies machine-wide (poll ledger, push dedup): its creating session
+	 * while that session is open — nobody else may take it — else the project's shared slot, so a
+	 * hand-over between processes still never double-checks.
+	 */
+	private slotOf(rule: DynamicTriggerRule): string {
+		const sid = rule.createdBy?.sessionId;
+		return sid && this.ownerOf(rule)?.sessionId === sid ? sid : "";
+	}
+
+	/**
+	 * This host's enabled rules that govern `cwd`, split into the ones this process owns. A rule
+	 * created at `~/proj` still governs a pi opened at `~/proj/src` or through a symlink to it:
+	 * comparing the cwd strings silently diverted those promotions to the inbox.
+	 *
+	 * `ownIfUnowned` is passed straight to `ownsRule`: it only adds the rules nobody owns.
+	 */
+	private rulesFor(cwd: string | undefined, ownIfUnowned: boolean): { applicable: DynamicTriggerRule[]; owned: DynamicTriggerRule[] } {
+		const host = this.self?.host ?? os.hostname();
+		const all = this.store.load().filter((r) => r.enabled && (!r.host || r.host === host));
+		const applicable = cwd ? all.filter((r) => withinProject(r.cwd, cwd)) : all;
+		return { applicable, owned: applicable.filter((r) => this.ownsRule(r, ownIfUnowned)) };
+	}
+
+	/**
+	 * Scheduler tick (every process): emit one periodic check per project whose rules this process
+	 * owns, at most once per poll interval machine-wide. The ledger is claimed per ownership slot,
+	 * not per project, so two windows in one repo each check their own rules once per interval while
+	 * a hand-over of the project's own rules still never double-checks.
+	 */
+	async tick(now: number, leader: boolean): Promise<void> {
+		// Before raising new work: the pushes this process refused while it was busy. They are its own
+		// backlog, not the leader's, so leadership does not gate them.
+		this.retryPending();
+		const host = this.self?.host ?? os.hostname();
+		const rules = this.store.load().filter((r) => r.enabled && (!r.host || r.host === host) && this.ownsRule(r, leader) && !this.inBackoff(r, now));
+		if (!rules.length) return;
+		const byCwd = new Map<string, DynamicTriggerRule[]>();
+		for (const r of rules) byCwd.set(r.cwd, [...(byCwd.get(r.cwd) ?? []), r]);
+		for (const [cwd, group] of byCwd) {
+			// A checkout that is gone cannot be checked. The scheduler disables such a cron job with
+			// an actionable message; without the same guard here the rules poll — and bill — forever.
+			if (cwd && !fs.existsSync(cwd)) {
+				for (const rule of group) {
+					await this.store.setEnabled(rule.id, false).catch(() => undefined);
+					this.store.appendAudit({ cwd, type: "trigger", traceId: newTraceId(), state: "disabled", sourceLabel: "local:dynamic", eventLabel: rule.id, summary: `cwd ${cwd} no longer exists`, details: { reason: "cwd missing" } });
+				}
+				this.log(`disabled ${group.length} rule(s): cwd ${cwd} no longer exists (re-enable after restoring it)`);
+				continue;
+			}
+			if ([...this.running.values()].some((r) => r.cwd === cwd && r.sourceLabel === "local:dynamic")) continue; // previous check still active
+			const bySlot = new Map<string, DynamicTriggerRule[]>();
+			for (const r of group) {
+				const slot = this.slotOf(r);
+				bySlot.set(slot, [...(bySlot.get(slot) ?? []), r]);
+			}
+			const claimed: DynamicTriggerRule[] = [];
+			for (const [slot, owned] of bySlot) {
+				if (await this.ledger.claim(`${host}:${cwd}${slot ? `#${slot}` : ""}`, now, this.pollIntervalSecs * 1000)) claimed.push(...owned);
+			}
+			if (!claimed.length) continue;
+			this.lastCheckAt = now;
+			// `handle` is documented not to reject; the catch is what makes that true rather than
+			// merely intended, since nothing above this would survive one.
+			void this.handle(buildPeriodicCheckTrigger(cwd, claimed.length, new Date(now), this.self ? `${this.self.pid}-${this.self.instance}` : ""), "sub_agent", claimed).catch((err: any) =>
+				this.log(`check failed: ${err?.message ?? err}`),
+			);
+		}
+	}
+
+	/**
+	 * Admit one trigger: dedup, audit, deliver. Resolves when the delivery has finished and
+	 * never rejects: persistence or runner failures are audited (best effort) and logged.
+	 */
+	async handle(trigger: Trigger, delivery: TriggerDelivery, rules?: DynamicTriggerRule[]): Promise<TriggerOutcome | undefined> {
+		// Sub-agents register no notification hooks and run no dynamic checker, so only the
+		// interactive process (hop 0) ever handles a trigger. Anything reaching a deeper hop is a
+		// cycle and is suppressed, audited as cycle-suppressed.
+		if (this.hop > 0) {
+			this.cycleSuppressedCount++;
+			this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger", traceId: trigger.traceId, state: "cycle_suppressed", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { hop_count: this.hop, delivery, ...envelopeOf(trigger) } });
+			this.log(`trigger ${trigger.traceId.slice(0, 8)} cycle_suppressed at hop ${this.hop} (${trigger.sourceLabel} / ${trigger.eventLabel})`);
+			return undefined;
+		}
+		try {
+			if (trigger.source.kind === "mcp" && delivery === "sub_agent" && !trigger.cwd && !this.getSession().cwd) {
+				// No project of our own (the headless host): evaluate the push once per project that has
+				// rules, each in that project, as tick() does — never all rules at once in $HOME.
+				const host = this.self?.host ?? os.hostname();
+				const cwds = [...new Set(this.store.load().filter((r) => r.enabled && (!r.host || r.host === host)).map((r) => r.cwd))];
+				if (!cwds.length) return await this.admit(trigger, delivery); // audited as no_rules
+				const outcomes = await Promise.all(cwds.map((cwd, i) => this.admit({ ...trigger, cwd, traceId: i ? newTraceId() : trigger.traceId }, delivery)));
+				return outcomes.find(Boolean);
+			}
+			return await this.admit(trigger, delivery, rules);
+		} catch (err: any) {
+			const message = err?.message ?? String(err);
+			this.running.delete(trigger.traceId);
+			this.log(`trigger ${trigger.traceId.slice(0, 8)} failed: ${message}`);
+			this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger_result", traceId: trigger.traceId, state: "failed", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: message, details: { delivery, error: message, ...envelopeOf(trigger) } });
+			return undefined;
+		}
+	}
+
+	private async admit(trigger: Trigger, delivery: TriggerDelivery, rules?: DynamicTriggerRule[]): Promise<TriggerOutcome | undefined> {
+		// A sub-agent costs money and a process slot. Both bounds are checked before the dedup claim,
+		// so a refused trigger is never marked handled — a periodic check is simply raised again by the
+		// next poll, and a push (an event that happened once, which no server repeats) is held below.
+		// `inject_summary` puts the push's own text into the chat and runs no model call — its audit
+		// row records `cost_usd: 0`. Refusing a free delivery for want of budget is backwards, and
+		// the day the cap trips is the day you still want to be told what is arriving.
+		const budget = this.budget();
+		const gate = delivery === "inject_summary" ? { over: false, spent: budget.spent, cap: budget.cap } : budget;
+		// Said out loud, because otherwise "why did this one arrive when everything else stopped" has
+		// no answer anywhere.
+		if (delivery === "inject_summary" && budget.over) this.log(`trigger ${trigger.traceId.slice(0, 8)} injected despite today's $${budget.cap.toFixed(2)} budget: an injected summary runs no model call`);
+		if (gate.over) {
+			// Not held: being too busy clears in minutes, but the daily cap can last until midnight,
+			// and acting on this morning's deploy event at 23:59 is worse than not acting at all.
+			this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger", traceId: trigger.traceId, state: "budget_exceeded", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { ...envelopeOf(trigger), delivery, spent_usd: gate.spent, cap_usd: gate.cap } });
+			this.log(`trigger ${trigger.traceId.slice(0, 8)} not run: today's automation has cost $${gate.spent.toFixed(2)} of the $${gate.cap.toFixed(2)} budget`);
+			return undefined;
+		}
+		// The counter is shared with loop runs and the /goal evaluator (src/slots.ts): the cap is on
+		// sub-agents, not on checks, so a machine full of loop runs refuses a check too. The slot is
+		// reserved here, before the dedup claim below, so the claim cannot outlive its admission.
+		let slot: SubagentSlot | undefined;
+		if (delivery === "sub_agent") {
+			slot = this.slots.acquire();
+			if (!slot) {
+				// Only a push is kept. Re-queueing a periodic check would be wrong: the next poll evaluates
+				// every rule against the world as it is then, so a held one could only run the same check
+				// twice. (`handle` has already fanned a project-less push out per project, so each entry
+				// here is one event for one project.)
+				const queued = trigger.source.kind === "mcp" ? this.queuePush(trigger, delivery) : undefined;
+				const { inUseCount, limit } = this.slots;
+				this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger", traceId: trigger.traceId, state: "deferred", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { ...envelopeOf(trigger), delivery, reason: `${inUseCount} sub-agent(s) already in flight (max ${limit})`, ...(queued ? { queued, pending: this.pending.size } : {}) } });
+				this.log(`trigger ${trigger.traceId.slice(0, 8)} deferred: ${inUseCount} sub-agent(s) already in flight${queued ? " (queued for the next tick)" : ""}`);
+				return undefined;
+			}
+		}
+		try {
+			let prev: Awaited<ReturnType<DedupWindow["check"]>>;
+			if (trigger.source.kind === "mcp" && delivery === "sub_agent") {
+				// A push evaluated against dynamic rules: by the process that owns them (each session
+				// evaluates its own registry), and once per ownership slot machine-wide.
+				const cwd = trigger.cwd ?? this.getSession().cwd;
+				const scope = this.rulesFor(cwd, this.isLeader?.() ?? true);
+				rules = scope.owned;
+				if (scope.applicable.length && !scope.owned.length) {
+					// The owner may not have this MCP server connected at all — servers authenticate per
+					// process — and a push nobody handles is simply lost. Record the hand-off, then take the
+					// push back if the owner has not claimed the slot below inside the takeover window; the
+					// claim is machine-wide, so at most one process ever evaluates it.
+					this.store.appendAudit({ cwd, type: "trigger", traceId: trigger.traceId, state: "deferred", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { delivery, reason: "another pi in this project owns rule evaluation", takeover_after_ms: this.deferredTakeoverMs, ...envelopeOf(trigger) } });
+					await new Promise((r) => setTimeout(r, this.deferredTakeoverMs));
+					rules = scope.applicable;
+				}
+				prev = await this.dedup.check(this.pushClaimKey(trigger, cwd, rules), trigger.traceId, this.now(), trigger.replacementPolicy);
+				if (!prev && !scope.owned.length && scope.applicable.length) {
+					this.store.appendAudit({ cwd, type: "trigger", traceId: trigger.traceId, state: "taken_over", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { delivery, reason: "the owning pi did not claim the push", ...envelopeOf(trigger) } });
+				}
+			} else if (trigger.source.kind === "mcp") {
+				// A push injected into the chat: every window that has the server reacts (every
+				// session), so the dedup window is this process's own.
+				prev = await this.localDedup.check(trigger.idempotencyKey, trigger.traceId, this.now(), trigger.replacementPolicy);
+			} else {
+				prev = await this.dedup.check(trigger.idempotencyKey, trigger.traceId, this.now(), trigger.replacementPolicy);
+			}
+			if (prev) {
+				this.dedupedCount++;
+				this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger", traceId: trigger.traceId, state: "deduped", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { previous_trace_id: prev.traceId, ...envelopeOf(trigger), replacement_policy: prev.replacementPolicy ?? trigger.replacementPolicy } });
+				// `[trigger deduped]` is a feed line; here the only trace was an audit row, so
+				// "my webhook fired and nothing happened" had no answer short of reading the JSONL.
+				this.log(`trigger ${trigger.traceId.slice(0, 8)} deduped (${trigger.sourceLabel} / ${trigger.eventLabel}): an identical event arrived within the dedup window`);
+				return undefined;
+			}
+			this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger", traceId: trigger.traceId, state: "accepted", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { delivery, evaluator_decision: { outcome: "accept", permission: "allow" }, ...envelopeOf(trigger) } });
+			// `return await`, not `return`: a bare return inside a try/finally hands the promise back and
+			// runs the finally at once, which would release the slot the moment the sub-agent started.
+			if (delivery === "inject_summary") return await this.deliverInjectSummary(trigger);
+			if (delivery === "inject_and_run") return await this.deliverInjectAndRun(trigger);
+			return await this.deliverSubAgent(trigger, rules);
+		} finally {
+			// Held for the whole delivery, `deliverSubAgent` included, and released on every path out of
+			// it — a torn dedup file, a runner that rejects, an abort — or the pool leaks a slot.
+			slot?.release();
+		}
+	}
+
+	/** When the event happened, which is what "oldest first" means for a push; `now` if the source sent us an unparseable stamp. */
+	private eventTimeOf(trigger: Trigger): number {
+		const at = Date.parse(trigger.receivedAt);
+		return Number.isNaN(at) ? this.now() : at;
+	}
+
+	/**
+	 * Hold a push that found no free slot. Keyed by idempotency key and project, so a server pushing
+	 * the same event again while we are busy collapses into the one already waiting instead of filling
+	 * the list; which copy survives is the envelope's own replacement policy, as in the dedup window.
+	 */
+	private queuePush(trigger: Trigger, delivery: TriggerDelivery): "queued" | "replaced" | "collapsed" {
+		const cwd = trigger.cwd ?? this.getSession().cwd;
+		const key = `${trigger.idempotencyKey}@${cwd}`;
+		const waiting = this.pending.get(key);
+		if (waiting) {
+			if (trigger.replacementPolicy !== "latest_replaces") return "collapsed";
+			this.pending.set(key, { trigger, delivery });
+			return "replaced";
+		}
+		if (this.pending.size >= MAX_PENDING_PUSHES) {
+			const oldest = [...this.pending.entries()].sort((a, b) => this.eventTimeOf(a[1].trigger) - this.eventTimeOf(b[1].trigger))[0];
+			this.pending.delete(oldest[0]);
+			const lost = oldest[1].trigger;
+			// The drop leaves a row of its own: "my webhook fired and nothing happened" must stay
+			// answerable from the audit, and the answer here is that the machine never caught up.
+			this.store.appendAudit({ cwd: lost.cwd ?? cwd, type: "trigger", traceId: lost.traceId, state: "dropped", sourceLabel: lost.sourceLabel, eventLabel: lost.eventLabel, summary: lost.payloadSummary, details: { ...envelopeOf(lost), delivery: oldest[1].delivery, reason: `${MAX_PENDING_PUSHES} pushes already waiting for a free slot` } });
+			this.log(`trigger ${lost.traceId.slice(0, 8)} dropped: ${MAX_PENDING_PUSHES} pushes already waiting for a free slot`);
+		}
+		this.pending.set(key, { trigger, delivery });
+		return "queued";
+	}
+
+	/**
+	 * Retry the held pushes, oldest event first, as many as there are free slots. Each goes back
+	 * through `handle`, so it is deduped, audited and delivered exactly as if it had just arrived —
+	 * a deferral, not a bypass — and carries how long it waited so the check can weigh its age.
+	 */
+	private retryPending(): void {
+		// A bound on how many are dequeued at once, not the admission test: `admit` reserves the slot
+		// itself, so a push dequeued past the free count is simply audited and queued again.
+		let slots = this.slots.free;
+		if (slots <= 0 || !this.pending.size) return;
+		for (const [key, entry] of [...this.pending.entries()].sort((a, b) => this.eventTimeOf(a[1].trigger) - this.eventTimeOf(b[1].trigger))) {
+			if (slots-- <= 0) break;
+			this.pending.delete(key);
+			const trigger: Trigger = { ...entry.trigger, deferredMs: Math.max(0, this.now() - this.eventTimeOf(entry.trigger)) };
+			// `handle` is documented not to reject; the catch is what makes that true rather than
+			// merely intended, since nothing above a tick would survive one.
+			void this.handle(trigger, entry.delivery).catch((err: any) => this.log(`deferred push ${trigger.traceId.slice(0, 8)} failed: ${err?.message ?? err}`));
+		}
+	}
+
+	/**
+	 * Whether `rule`'s next poll is still inside its failure backoff. A check that has failed three
+	 * times in a row is failing for a reason ten minutes will not fix, and every attempt bills a
+	 * sub-agent; pushes still reach the rule, because an event is worth one attempt.
+	 */
+	private inBackoff(rule: DynamicTriggerRule, now: number): boolean {
+		const failures = rule.consecutiveFailures ?? 0;
+		if (failures < CHECK_FAILURE_BACKOFF_AFTER || !rule.lastCheckFailedAt) return false;
+		const wait = backoffWaitMs(failures, CHECK_FAILURE_BACKOFF_AFTER, CHECK_FAILURE_BACKOFF_BASE_MS, CHECK_FAILURE_BACKOFF_MAX_MS);
+		return now - Date.parse(rule.lastCheckFailedAt) < wait;
+	}
+
+	/** Record a check's outcome on the rules it evaluated, and say in the audit when that starts holding a poll back. */
+	private async recordCheckOutcome(rules: DynamicTriggerRule[], ok: boolean, cwd: string): Promise<void> {
+		const ids = rules.map((r) => r.id);
+		if (ok && !rules.some((r) => r.consecutiveFailures)) return; // nothing to clear: no lock, no write
+		const nowIso = stamp(this.now());
+		const failed = await this.store.mutate((all) => {
+			const out: DynamicTriggerRule[] = [];
+			for (const rule of all) {
+				if (!ids.includes(rule.id)) continue;
+				if (ok) {
+					rule.consecutiveFailures = undefined;
+					rule.lastCheckFailedAt = undefined;
+					continue;
+				}
+				rule.consecutiveFailures = (rule.consecutiveFailures ?? 0) + 1;
+				rule.lastCheckFailedAt = nowIso;
+				out.push(rule);
+			}
+			return out;
+		});
+		const backing = failed.filter((r) => (r.consecutiveFailures ?? 0) >= CHECK_FAILURE_BACKOFF_AFTER);
+		if (!backing.length) return;
+		const worst = Math.max(...backing.map((r) => r.consecutiveFailures ?? 0));
+		this.store.appendAudit({ cwd, type: "trigger", traceId: newTraceId(), state: "backoff", sourceLabel: "local:dynamic", eventLabel: backing.map((r) => r.id).join(" "), summary: `${backing.length} rule(s) checked with a widening gap after ${worst} failed check(s) in a row`, details: { rule_ids: backing.map((r) => r.id), consecutive_failures: worst } });
+		this.log(`${backing.length} trigger rule(s) failed ${worst} checks in a row; polling them with a widening gap`);
+	}
+
+	/**
+	 * The machine-wide claim a pushed evaluation takes. Keyed by the project (realpath, so two
+	 * windows reaching it by different paths agree) and by the ownership slots of the rules being
+	 * evaluated, so windows checking *different* rules of one project both react while a deferred
+	 * push and its owner still contend for the same key.
+	 */
+	private pushClaimKey(trigger: Trigger, cwd: string, rules: DynamicTriggerRule[]): string {
+		const slots = [...new Set(rules.map((r) => this.slotOf(r)))].sort().join(",");
+		return `${trigger.idempotencyKey}@${this.self?.host ?? os.hostname()}:${realpathish(cwd)}${slots ? `#${slots}` : ""}`;
+	}
+
+	private async deliverInjectSummary(trigger: Trigger): Promise<TriggerOutcome> {
+		const start = this.now();
+		const summary = trigger.payloadSummary ?? "";
+		let promoted = false;
+		if (summary) {
+			const target = (await this.hooks.onPromote?.(promotionBody(trigger, summary), trigger)) ?? "chat";
+			promoted = target === "chat";
+			this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger_promotion", traceId: trigger.traceId, state: target === "chat" ? "promoted" : "redirected", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary, details: { prefix_injected: true, delivery: "inject_summary", to: target, ...envelopeOf(trigger) } });
+		}
+		// `over_budget` on a completed row is the audit's answer to "everything else stopped today —
+		// why did this run": because it cost nothing.
+		const spend = this.budget();
+		this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger_result", traceId: trigger.traceId, state: "completed", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary, details: { delivery: "inject_summary", cost_usd: 0, ...(spend.over ? { over_budget: true } : {}), ...envelopeOf(trigger) } });
+		const outcome: TriggerOutcome = { trigger, delivery: "inject_summary", ok: true, matchedRules: [], summary, durationMs: this.now() - start, cost: 0, promoted };
+		this.hooks.onFinished?.(outcome);
+		return outcome;
+	}
+
+	private async deliverInjectAndRun(trigger: Trigger): Promise<TriggerOutcome> {
+		const start = this.now();
+		const prompt = `[Trigger ${trigger.traceId}] ${trigger.payloadSummary ?? `${trigger.sourceLabel} fired: ${trigger.eventLabel}`}`;
+		const target = (await this.hooks.onInjectAndRun?.(prompt, trigger)) ?? "chat";
+		this.store.appendAudit({ cwd: trigger.cwd ?? this.getSession().cwd, type: "trigger_result", traceId: trigger.traceId, state: "completed", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { delivery: "inject_and_run", prefix_injected: true, cost_usd: 0, to: target, ...envelopeOf(trigger) } });
+		const outcome: TriggerOutcome = { trigger, delivery: "inject_and_run", ok: true, matchedRules: [], summary: trigger.payloadSummary ?? "", durationMs: this.now() - start, cost: 0, promoted: target === "chat" };
+		this.hooks.onFinished?.(outcome);
+		return outcome;
+	}
+
+	/**
+	 * The project an audit row of this trigger belongs to. Resolved once per delivery: a check takes
+	 * minutes, and the session's cwd can move under it (`/resume`, a project switch), which used to
+	 * split one trace across two projects in `/triggers audit`.
+	 */
+	private cwdOf(trigger: Trigger): string {
+		return trigger.cwd ?? this.getSession().cwd;
+	}
+
+	private async deliverSubAgent(trigger: Trigger, evaluate?: DynamicTriggerRule[]): Promise<TriggerOutcome> {
+		const start = this.now();
+		const session = this.getSession();
+		const cwd = this.cwdOf(trigger);
+		// `evaluate` is what the caller already resolved (the tick's claimed rules, a taken-over
+		// push); otherwise this is a direct handle() and we take the rules of this project we own.
+		const rules = evaluate ?? this.rulesFor(trigger.cwd, this.isLeader?.() ?? true).owned;
+		if (!rules.length) {
+			this.store.appendAudit({ cwd, type: "trigger_result", traceId: trigger.traceId, state: "no_rules", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: trigger.payloadSummary, details: { delivery: "sub_agent", ...envelopeOf(trigger) } });
+			const outcome: TriggerOutcome = { trigger, delivery: "sub_agent", ok: true, matchedRules: [], summary: "no enabled dynamic trigger rules", durationMs: 0, cost: 0, promoted: false };
+			this.hooks.onFinished?.(outcome);
+			return outcome;
+		}
+		const prompt = renderDynamicTriggerPrompt(trigger, rules);
+		const ctrl = new AbortController();
+		const running: RunningTrigger = { traceId: trigger.traceId, sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, startedAt: stamp(start), promptPreview: previewRedacted(prompt, 80), cwd, ctrl }; // 80 characters is a banner, not a prompt
+		this.running.set(trigger.traceId, running);
+		this.hooks.onStarted?.(running);
+		this.store.appendAudit({ cwd, type: "trigger_result", traceId: trigger.traceId, state: "running", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, details: { rule_count: rules.length, cwd, ...envelopeOf(trigger) } });
+
+		// The check runs with the model the rules were created under, not whatever the process that
+		// happens to own the timer is using. Both come from the first rule that recorded a model,
+		// because a thinking level belongs to the model it was chosen for: pairing them with a level
+		// from some other rule is how a check asks for a budget the model does not have.
+		const withModel = rules.find((r) => r.model);
+		const model = withModel?.model ?? session.model;
+		const thinking = withModel?.thinking ?? session.thinking;
+		let result: RunnerResult;
+		try {
+			// A check runs every rule of the project, so the longest per-rule cap wins.
+			const timeoutMs = rules.reduce((max, r) => Math.max(max, r.timeoutMs ?? 0), 0) || this.runTimeoutMs;
+			result = await this.runner({ cwd, prompt, model, thinking, timeoutMs, signal: ctrl.signal, sessionDir: this.jobStore.sessionDirFor(triggerSessionKey(cwd)), hop: this.hop + 1, parentSessionId: session.sessionId, parentCwd: session.cwd, kind: "trigger", traceId: trigger.traceId });
+		} catch (err: any) {
+			result = failedRun(err?.message ?? String(err));
+		} finally {
+			this.running.delete(trigger.traceId);
+		}
+		// Per project, not one budget for the machine: three projects polling every ten minutes used
+		// to exhaust a shared 40 within a couple of hours, taking the evidence with them. The key has
+		// to be the one the transcripts were written under, or the prune looks in a directory that
+		// does not exist and silently keeps everything.
+		this.jobStore.pruneSessions(triggerSessionKey(cwd), TRIGGER_SESSIONS_KEPT);
+
+		const summary = result.text.trim();
+		// A check killed by the run timeout (or aborted) has usually already *executed* the matching
+		// rule's action — posting the comment, kicking the deploy — so the matched ids in whatever it
+		// managed to say still have to disarm a fire-once rule; otherwise the next poll does it again.
+		const matchedIds = extractDynamicRuleIds(summary);
+		const matchedRules = rules.filter((r) => matchedIds.includes(r.id));
+		if (matchedRules.length) await this.store.markFired(matchedRules.map((r) => r.id));
+		const quiet = result.ok && matchedRules.length === 0;
+		const state = ctrl.signal.aborted ? "aborted" : result.ok ? "completed" : "failed";
+		this.store.appendAudit({
+			cwd,
+			type: "trigger_result",
+			traceId: trigger.traceId,
+			state,
+			sourceLabel: trigger.sourceLabel,
+			eventLabel: trigger.eventLabel,
+			summary: result.ok ? summary || NO_MATCH_SENTINEL : result.errorMessage,
+			details: { delivery: "sub_agent", matched_rule_ids: matchedIds, quiet, cost_usd: result.usage.cost, exit_code: result.exitCode, session_file: result.sessionFile, ...envelopeOf(trigger) },
+		});
+		// An abort is the user's decision about this run, not evidence that the rule's check is broken,
+		// so it neither counts against the rule nor clears what earlier failures recorded.
+		if (!ctrl.signal.aborted) await this.recordCheckOutcome(rules, result.ok, cwd);
+
+		let promoted = false;
+		const promoteRules = matchedRules.filter((r) => r.promoteToChat);
+		if (result.ok && promoteRules.length) {
+			const target = (await this.hooks.onPromote?.(promotionSummaryBody(trigger, summary), trigger)) ?? "chat";
+			promoted = target === "chat";
+			this.store.appendAudit({ cwd, type: "trigger_promotion", traceId: trigger.traceId, state: target === "chat" ? "promoted" : "redirected", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary, details: { prefix_injected: true, rule_ids: promoteRules.map((r) => r.id), to: target, template_name: "default", ...envelopeOf(trigger) } });
+		} else if (result.ok && matchedRules.length) {
+			this.store.appendAudit({ cwd, type: "trigger_promotion", traceId: trigger.traceId, state: "skipped", sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, details: { reason: "no matched rule has promote_to_chat", ...envelopeOf(trigger) } });
+		}
+		if (trigger.sourceLabel === "local:dynamic") this.lastPoll = { at: stamp(this.now()), cwd, outcome: state === "completed" ? (quiet ? "no match" : `matched ${matchedRules.length}`) : state, traceId: trigger.traceId, sourceLabel: trigger.sourceLabel, eventLabel: trigger.eventLabel, summary: previewRedacted(result.ok ? summary || NO_MATCH_SENTINEL : (result.errorMessage ?? ""), 160) };
+		const outcome: TriggerOutcome = { trigger, delivery: "sub_agent", ok: result.ok, matchedRules, summary, error: result.ok ? undefined : result.errorMessage, durationMs: this.now() - start, cost: result.usage.cost, promoted, sessionFile: result.sessionFile };
+		this.hooks.onFinished?.(outcome);
+		return outcome;
+	}
+}