cursedbelt 2.7.0 → 3.0.0

package/src/server/incidents/channels.ts

@@ -1,487 +0,0 @@
-/**
- * How an incident reaches the owner. Three channels, and a hard rule about which
- * incidents are allowed which.
- *
- * ── The severity split is the feature ───────────────────────────────────────
- * The owner asked for "an email and a text message about blocking issues like
- * these (can't deploy, can't ingest files)". A warning that texts him is a
- * warning that gets the channel muted, and a muted channel puts him back where
- * this whole unit started — finding out days later, by accident. So: in-app for
- * everything, email + text for `blocking` only, and `notifier.ts` is the only
- * place that decision is made.
- *
- * ── 🔴 "Sent" is a claim; this module deals in evidence ─────────────────────
- * `fleet/rules/automation-over-documentation.md` — *"treat 'I could not prove it
- * worked' as a failure rather than a pass"*. Both remote channels here can
- * succeed loudly and deliver nothing:
- *
- *  - **iMessage.** `osascript` exits 0 the moment Messages accepts the string.
- *    Whether anything left the machine is a separate question, and the answer is
- *    in `~/Library/Messages/chat.db` — so {@link createTextChannel} reads back
- *    an outgoing row with `is_sent = 1` before it will call the send delivered.
- *    Without that read it is `unproven`, which is NOT success.
- *  - **Email.** The fleet's mailer is best-effort by contract and returns
- *    `false` on both "mail is off here" and "SES refused". That is right for a
- *    signup mail and wrong for an outage alert, so the result is carried, not
- *    swallowed, and the console shows it on the incident.
- *
- * ── Why the text channel is AppleScript and not an SMS API ──────────────────
- * Measured on this Mac, 2026-08-20, while building this: neither IAM user the
- * console holds can call SES (`satellite-cloud` and `satellite-console-prod`
- * both answer `AccessDenied` on `ses:GetSendQuota`), and the Gmail refresh token
- * in `openclaw-digest.env` is revoked (`invalid_grant`). Messages.app is signed
- * in, has an enabled iMessage account, and its Automation grant is already in
- * place — one `osascript` reached the owner's phone in under a second with
- * `is_sent = 1, is_delivered = 1` in `chat.db`. It costs nothing, needs no
- * credential, and — the property that decides it — **it does not depend on the
- * prod box**, which is exactly the thing most likely to be down when an alert
- * matters. An SMS provider would have been a new bill and a new secret to reach
- * the same phone less reliably.
- */
-
-/** What a channel did. `unproven` is deliberately distinct from `delivered`. */
-export type ChannelOutcome = "delivered" | "unproven" | "skipped" | "failed";
-
-export interface ChannelResult {
-	channel: string;
-	outcome: ChannelOutcome;
-	/** One line, for the incident row and the console. Never carries the message body. */
-	detail: string;
-}
-
-/** One line of a notification. Channels render it themselves — see `notifier.ts`. */
-export interface ChannelMessage {
-	subject: string;
-	/** Plain text. Short enough for a phone; the console holds the full record. */
-	text: string;
-	/** Longer HTML, for email only. Optional — a channel may ignore it. */
-	html?: string;
-	/** `blocking` or `warning`, so a channel may refuse what it is not for. */
-	severity: "blocking" | "warning";
-	/**
-	 * Channels this message must NOT use, by key. The notifier sets it when the
-	 * message is ABOUT a channel being broken — telling somebody down a wire that
-	 * the wire is down is the loop this field exists to cut. **Enforced in
-	 * `notifier.ts`, once**, so a channel added later cannot forget it.
-	 */
-	avoid?: readonly string[];
-}
-
-export interface IncidentChannel {
-	readonly key: string;
-	/** Human name for the console's channel-health row. */
-	readonly label: string;
-	send: (message: ChannelMessage) => Promise<ChannelResult>;
-}
-
-// ── text: iMessage from this Mac ──────────────────────────────────────────────
-
-/**
- * The AppleScript, as a file-less `osascript -` script taking `argv`, so no part
- * of the message is ever interpolated into source. A message containing a quote
- * would otherwise be a syntax error at best.
- *
- * It walks the accounts by index inside a `try` rather than asking for
- * `1st account whose service type = iMessage`: on this Mac four of the six
- * accounts throw `-1728` on any property access, and the `whose` form propagates
- * that instead of skipping them (measured 2026-08-20 — the clean-looking form is
- * the one that does not work).
- */
-const SEND_SCRIPT = `on run argv
-	set targetHandle to item 1 of argv
-	set msg to item 2 of argv
-	tell application "Messages"
-		set n to (count of accounts)
-		repeat with i from 1 to n
-			try
-				set acct to account i
-				if (service type of acct) is iMessage and (enabled of acct) then
-					send msg to participant targetHandle of acct
-					return "sent"
-				end if
-			end try
-		end repeat
-	end tell
-	return "no-imessage-account"
-end run`;
-
-/**
- * What macOS has actually decided about this process talking to Messages.
- *
- * The user's TCC database is readable with Full Disk Access, which this Mac's
- * agents have, and it is the only place the answer exists — `osascript` reports
- * a timeout, not a refusal. Unreadable ⇒ `unknown`, never a guess.
- */
-async function automationDecision(): Promise<"denied" | "allowed" | "unknown"> {
-	try {
-		const { Database } = await import("bun:sqlite");
-		const { homedir } = await import("node:os");
-		const { join } = await import("node:path");
-		const db = new Database(
-			join(homedir(), "Library", "Application Support", "com.apple.TCC", "TCC.db"),
-			{ readonly: true },
-		);
-		try {
-			const row = db
-				.prepare(
-					`SELECT auth_value FROM access
-						WHERE service = 'kTCCServiceAppleEvents'
-							AND indirect_object_identifier = 'com.apple.MobileSMS'
-							AND client LIKE '%bun%'
-						ORDER BY last_modified DESC LIMIT 1`,
-				)
-				.get() as { auth_value: number } | null;
-			if (!row) return "unknown";
-			return row.auth_value === 0 ? "denied" : "allowed";
-		} finally {
-			db.close();
-		}
-	} catch {
-		return "unknown";
-	}
-}
-
-/** Apple's epoch is 2001-01-01; `message.date` is nanoseconds since then. */
-const APPLE_EPOCH_OFFSET_S = 978_307_200;
-const appleTimeNow = (): number => (Date.now() / 1000 - APPLE_EPOCH_OFFSET_S) * 1e9;
-
-export interface TextChannelOptions {
-	/** The handle to text — a phone number in E.164, or an iMessage address. */
-	handle: string;
-	/** Overridable for tests. Default: run `osascript`. */
-	send?: (handle: string, body: string) => Promise<{ ok: boolean; detail: string }>;
-	/**
-	 * Overridable for tests. Default: read `~/Library/Messages/chat.db` for an
-	 * outgoing row newer than `sinceAppleTime`. Returns null when the database
-	 * cannot be read at all — which is `unproven`, not `failed`.
-	 */
-	proveSent?: (sinceAppleTime: number) => Promise<boolean | null>;
-	/** How long to wait for the proof row. Default 8s; one arrived in <1s when measured. */
-	proofTimeoutMs?: number;
-}
-
-const defaultSend = async (
-	handle: string,
-	body: string,
-): Promise<{ ok: boolean; detail: string }> => {
-	const proc = Bun.spawn(["osascript", "-", handle, body], {
-		stdin: new TextEncoder().encode(SEND_SCRIPT),
-		stdout: "pipe",
-		stderr: "pipe",
-	});
-	const [out, err, code] = await Promise.all([
-		new Response(proc.stdout).text(),
-		new Response(proc.stderr).text(),
-		proc.exited,
-	]);
-	const trimmed = out.trim();
-	if (code !== 0) {
-		// 🔴 Two different macOS errors mean the same thing, and neither says so.
-		// `-1743` is an outright refusal. `-1712` is an AppleEvent TIMEOUT, which
-		// is what a DAEMON gets: the consent prompt is raised against the launchd
-		// job's responsible process, nobody answers it, and the event times out.
-		// Measured on this Mac 2026-08-21 — the console's first send produced
-		// `-1712`, and `TCC.db` had a fresh `kTCCServiceAppleEvents` row for
-		// `bun.exe → com.apple.MobileSMS` with `auth_value = 0` stamped at the same
-		// minute. So the raw error is a red herring and the decision is a FACT that
-		// can be read, which is what `automationDecision` does. Naming the state
-		// beats printing an error code nobody can act on.
-		const looksLikeTcc =
-			err.includes("-1743") ||
-			err.includes("-1712") ||
-			err.toLowerCase().includes("not authorized");
-		if (looksLikeTcc) {
-			const decision = await automationDecision();
-			return {
-				ok: false,
-				detail:
-					decision === "denied"
-						? "macOS has Automation access to Messages DENIED for this console's daemon. Turn it back on in System Settings ▸ Privacy & Security ▸ Automation (the entry is 'bun'), then press Send a test."
-						: "macOS did not grant Automation access to Messages — a consent prompt was raised against a background daemon and nothing answered it. Approve it in System Settings ▸ Privacy & Security ▸ Automation, then press Send a test.",
-			};
-		}
-		return { ok: false, detail: `osascript exited ${code}: ${err.trim().slice(0, 200)}` };
-	}
-	if (trimmed !== "sent") {
-		return { ok: false, detail: `Messages has no enabled iMessage account (${trimmed})` };
-	}
-	return { ok: true, detail: "Messages accepted the send" };
-};
-
-const defaultProveSent = async (sinceAppleTime: number): Promise<boolean | null> => {
-	try {
-		const { Database } = await import("bun:sqlite");
-		const { homedir } = await import("node:os");
-		const { join } = await import("node:path");
-		// Read-only, and never written: this is Apple's database, not ours.
-		const db = new Database(join(homedir(), "Library", "Messages", "chat.db"), {
-			readonly: true,
-		});
-		try {
-			const row = db
-				.prepare(
-					"SELECT COUNT(*) AS n FROM message WHERE is_from_me = 1 AND is_sent = 1 AND date > ?",
-				)
-				.get(sinceAppleTime) as { n: number } | null;
-			return (row?.n ?? 0) > 0;
-		} finally {
-			db.close();
-		}
-	} catch {
-		// Full Disk Access is not granted, or the schema moved. We cannot prove
-		// anything — which is `unproven`, and the caller must not read it as a send
-		// that failed OR as one that worked.
-		return null;
-	}
-};
-
-/**
- * Text the owner from this Mac, and prove it left. Mac-only by construction; the
- * prod deployment of the console never registers it (see `notifier.ts`).
- */
-export function createTextChannel(options: TextChannelOptions): IncidentChannel {
-	const send = options.send ?? defaultSend;
-	const prove = options.proveSent ?? defaultProveSent;
-	const timeoutMs = options.proofTimeoutMs ?? 8_000;
-
-	return {
-		key: "text",
-		label: "Text message (iMessage from this Mac)",
-		async send(message) {
-			if (message.severity !== "blocking") {
-				return { channel: "text", outcome: "skipped", detail: "not a blocking incident" };
-			}
-			const before = appleTimeNow();
-			const attempt = await send(options.handle, `${message.subject}\n\n${message.text}`);
-			if (!attempt.ok) return { channel: "text", outcome: "failed", detail: attempt.detail };
-
-			const deadline = Date.now() + timeoutMs;
-			let proofAvailable = true;
-			while (Date.now() < deadline) {
-				const proved = await prove(before);
-				if (proved === null) {
-					proofAvailable = false;
-					break;
-				}
-				if (proved) {
-					return { channel: "text", outcome: "delivered", detail: "chat.db shows it sent" };
-				}
-				await Bun.sleep(500);
-			}
-			return {
-				channel: "text",
-				outcome: "unproven",
-				detail: proofAvailable
-					? `Messages accepted it but no sent row appeared within ${Math.round(timeoutMs / 1000)}s`
-					: "Messages accepted it; chat.db is unreadable (Full Disk Access), so delivery is unproven",
-			};
-		},
-	};
-}
-
-// ── email ─────────────────────────────────────────────────────────────────────
-
-/** The one thing this module needs from a mail sender. Matches the fleet's `Mailer`. */
-export type SendEmail = (params: {
-	to: string;
-	subject: string;
-	text: string;
-	html: string;
-	template?: string;
-}) => Promise<boolean>;
-
-export interface EmailChannelOptions {
-	to: string;
-	send: SendEmail;
-	/**
-	 * Why this deployment might not be able to send — surfaced verbatim when the
-	 * send returns false, because "false" from a best-effort mailer means both
-	 * "mail is off on this host" and "SES refused" and the owner needs to know
-	 * which. E.g. "MAIL_ON is unset on the local-mac console".
-	 */
-	unavailableHint?: string;
-}
-
-export function createEmailChannel(options: EmailChannelOptions): IncidentChannel {
-	return {
-		key: "email",
-		label: `Email to ${options.to}`,
-		async send(message) {
-			if (message.severity !== "blocking") {
-				return { channel: "email", outcome: "skipped", detail: "not a blocking incident" };
-			}
-			try {
-				const ok = await options.send({
-					to: options.to,
-					subject: message.subject,
-					text: message.text,
-					html: message.html ?? `<pre>${escapeHtml(message.text)}</pre>`,
-					template: "fleet.incident",
-				});
-				return ok
-					? { channel: "email", outcome: "delivered", detail: "accepted by the mailer" }
-					: {
-							channel: "email",
-							outcome: "failed",
-							detail: options.unavailableHint ?? "the mailer declined to send",
-						};
-			} catch (err) {
-				return {
-					channel: "email",
-					outcome: "failed",
-					detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
-				};
-			}
-		},
-	};
-}
-
-/**
- * Email, sent by SOMEBODY ELSE. The Mac cannot reach SES (see the module
- * header), and the prod box can — so the local console hands the message to the
- * prod console over the wire the two already share.
- *
- * 🔴 It is a fallback and never the only path. The failure mode it cannot cover
- * is the one where prod is down, which is precisely when an alert matters most —
- * that case belongs to the text channel, which runs entirely on this Mac.
- */
-export function createRelayEmailChannel(options: {
-	/** e.g. `https://admin.cursedalchemy.com/api/station/incidents/mail/relay-email`. */
-	url: string;
-	/** Shared bearer — the console's existing sync token. */
-	token: string;
-	/**
-	 * 🔴 Headers the EDGE needs, distinct from the bearer the app needs.
-	 *
-	 * The prod console sits behind a Cloudflare Access application, so a plain
-	 * bearer request never reaches the origin at all — Access answers 302 to its
-	 * own login page. Measured 2026-08-21: `fetch` FOLLOWED that redirect, got a
-	 * 200 of HTML, and this channel reported "the prod console accepted the
-	 * request but its mailer declined". Every word of that was wrong and it named
-	 * the wrong host's component. The caller supplies the service-token headers
-	 * (satellite-kit's `accessHeadersForUrl`); this package must not know about
-	 * Cloudflare.
-	 */
-	edgeHeaders?: () => Record<string, string>;
-	fetchImpl?: typeof fetch;
-	timeoutMs?: number;
-}): IncidentChannel {
-	const doFetch = options.fetchImpl ?? fetch;
-	return {
-		key: "email",
-		label: "Email (relayed through the prod console)",
-		async send(message) {
-			if (message.severity !== "blocking") {
-				return { channel: "email", outcome: "skipped", detail: "not a blocking incident" };
-			}
-			try {
-				const res = await doFetch(options.url, {
-					method: "POST",
-					headers: {
-						"content-type": "application/json",
-						authorization: `Bearer ${options.token}`,
-						...(options.edgeHeaders?.() ?? {}),
-					},
-					body: JSON.stringify({
-						subject: message.subject,
-						text: message.text,
-						html: message.html,
-					}),
-					// 🔴 `manual`, so an edge redirect is a FAILURE rather than a
-					// successful fetch of a login page. Following it is what turned an
-					// Access refusal into a 200 and a wrong diagnosis.
-					redirect: "manual",
-					signal: AbortSignal.timeout(options.timeoutMs ?? 15_000),
-				});
-				if (res.status >= 300 && res.status < 400) {
-					return {
-						channel: "email",
-						outcome: "failed",
-						detail:
-							`the request was redirected (${res.status}) before it reached the console — ` +
-							"an edge access gate is refusing it, so the service-token headers are missing or expired",
-					};
-				}
-				if (!res.ok) {
-					return {
-						channel: "email",
-						outcome: "failed",
-						detail: `the prod console answered ${res.status}`,
-					};
-				}
-				const body = (await res.json().catch(() => null)) as {
-					sent?: boolean;
-					reason?: string;
-				} | null;
-				return body?.sent
-					? { channel: "email", outcome: "delivered", detail: "sent by the prod console" }
-					: {
-							channel: "email",
-							outcome: "failed",
-							// The relaying host is the only one that knows why, so it says so.
-							detail: body?.reason
-								? `the prod console could not send it: ${body.reason}`
-								: "the prod console accepted the request but its mailer declined, and gave no reason",
-						};
-			} catch (err) {
-				return {
-					channel: "email",
-					outcome: "failed",
-					detail: `could not reach the prod console: ${
-						err instanceof Error ? err.message.slice(0, 160) : String(err)
-					}`,
-				};
-			}
-		},
-	};
-}
-
-// ── in-app ────────────────────────────────────────────────────────────────────
-
-/**
- * The in-app channel is the fleet's EXISTING notification primitive
- * (`cursedbelt/server/notifications` over `cwip/notifications`), not a second
- * inbox. Pass its `notify` and the incident's fingerprint becomes the
- * notification's `groupKey`, so the database's partial unique index coalesces
- * repeats for free — the same guarantee the store makes, enforced a second time
- * one layer down.
- */
-export function createInAppChannel(options: {
-	notify: (input: {
-		title: string;
-		body: string;
-		groupKey: string;
-		severity: "blocking" | "warning";
-	}) => Promise<void> | void;
-	/** The fingerprint of the incident being announced. Set per-message by the notifier. */
-	groupKeyOf: (message: ChannelMessage) => string;
-}): IncidentChannel {
-	return {
-		key: "in-app",
-		label: "The console's notification bell",
-		async send(message) {
-			try {
-				await options.notify({
-					title: message.subject,
-					body: message.text,
-					groupKey: options.groupKeyOf(message),
-					severity: message.severity,
-				});
-				return { channel: "in-app", outcome: "delivered", detail: "filed in the inbox" };
-			} catch (err) {
-				return {
-					channel: "in-app",
-					outcome: "failed",
-					detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
-				};
-			}
-		},
-	};
-}
-
-const escapeHtml = (s: string): string =>
-	s.replace(
-		/[&<>"']/g,
-		(c) =>
-			({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c] as string,
-	);

package/src/server/incidents/checks.ts

@@ -1,120 +0,0 @@
-/**
- * The check contract and the sweep that runs it.
- *
- * A check is a pure-ish function from "the machine" to "the conditions that are
- * true right now". It does not know whether any of them are new, it never
- * notifies, and it cannot resolve anything — all three belong to the store, so
- * that one implementation of the dedupe rule covers every detector anybody adds
- * later. Adding a detector is therefore a function and a row, not a design.
- *
- * ── 🔴 A check that FAILS resolves nothing ──────────────────────────────────
- * This is the invariant the whole feature's credibility rests on. If a detector
- * throws — its command is missing, the disk it stats is unmounted, a fetch times
- * out — the sweep records the failure and SKIPS the reconcile for that check.
- * Reconciling would have handed the store an empty report set, and an empty
- * report set from an authoritative check means "all clear", which would send the
- * owner a cheerful "cleared: production is down" the moment the detector broke.
- * The same reasoning applies one level up, in `runIncidentSweep`'s return value:
- * a sweep reports its own failures so the console can show a detector that has
- * stopped working, rather than showing an empty incident list and looking calm.
- */
-import type { IncidentStore } from "./store";
-import type { IncidentReport, IncidentTransitions } from "./types";
-
-export interface IncidentCheck {
-	/** Stable key. Owns every incident it has ever opened — see `reconcile`. */
-	readonly key: string;
-	/** The app or service this check speaks for: `binary-server`, `station`, `orch`. */
-	readonly source: string;
-	/**
-	 * Every `kind` this check can produce. Declaring them is what lets a run that
-	 * finds nothing RESOLVE its own past incidents — see `ReconcileOptions`.
-	 * A kind missing from this list can be opened but never automatically cleared,
-	 * so the list is not optional decoration.
-	 */
-	readonly kinds: readonly string[];
-	/** Human name, for the console's detector table. */
-	readonly label: string;
-	run: () => Promise<IncidentReport[]> | IncidentReport[];
-}
-
-export interface CheckOutcome {
-	key: string;
-	label: string;
-	/** `ok` when it ran, whatever it found. `failed` when it threw. */
-	status: "ok" | "failed";
-	reported: number;
-	durationMs: number;
-	error?: string;
-}
-
-export interface SweepResult {
-	at: number;
-	outcomes: CheckOutcome[];
-	transitions: IncidentTransitions;
-}
-
-const emptyTransitions = (): IncidentTransitions => ({
-	opened: [],
-	escalated: [],
-	resolved: [],
-	unchanged: [],
-});
-
-/**
- * Run every check once, reconcile each one's results independently, and return
- * the union of the transitions plus a per-check outcome.
- *
- * Checks run in PARALLEL (they are mostly I/O — a stat, a fetch, a sqlite read)
- * but reconcile in SERIES against one sqlite handle, which keeps the store's
- * writes single-threaded without needing a transaction across unrelated checks.
- */
-export async function runIncidentSweep(
-	store: IncidentStore,
-	checks: readonly IncidentCheck[],
-	now: number = Date.now(),
-): Promise<SweepResult> {
-	const settled = await Promise.all(
-		checks.map(async (check) => {
-			const started = Date.now();
-			try {
-				const reports = await check.run();
-				return { check, reports, error: null as string | null, durationMs: Date.now() - started };
-			} catch (err) {
-				return {
-					check,
-					reports: [] as IncidentReport[],
-					error: err instanceof Error ? err.message.slice(0, 300) : String(err),
-					durationMs: Date.now() - started,
-				};
-			}
-		}),
-	);
-
-	const transitions = emptyTransitions();
-	const outcomes: CheckOutcome[] = [];
-
-	for (const { check, reports, error, durationMs } of settled) {
-		outcomes.push({
-			key: check.key,
-			label: check.label,
-			status: error ? "failed" : "ok",
-			reported: reports.length,
-			durationMs,
-			...(error ? { error } : {}),
-		});
-		// The invariant in the header: a failed check reconciles nothing at all.
-		if (error) continue;
-
-		const t = store.reconcile(check.key, reports, {
-			namespaces: check.kinds.map((k) => `${check.source}:${k}:`),
-			now,
-		});
-		transitions.opened.push(...t.opened);
-		transitions.escalated.push(...t.escalated);
-		transitions.resolved.push(...t.resolved);
-		transitions.unchanged.push(...t.unchanged);
-	}
-
-	return { at: now, outcomes, transitions };
-}