@dumbmice/pi-mailbridge 0.0.0-stage → 0.1.1

package/src/daemon/process.ts

@@ -0,0 +1,348 @@
+// Scan → fetch → authorize → parse → persist. Port of opencode-afk `core/process.js`.
+//
+// THE ORDERING HERE IS THE CORRECTNESS PROPERTY (docs/ARCHITECTURE.md §2):
+//
+//   0. skip everything if mode is off       `/mailbridge off` must close the channel, not just mute it
+//   1. search `UID <cursor>+1:*`            UID cursor, never \Seen, never SUBJECT
+//   2. fetch envelope + body
+//   3. self-echo, part 1: the marker header (before authorization, because in single-mailbox
+//      mode the sender IS the human and would otherwise pass every later gate)
+//   4. authorize (loop → robot → allowlist → DMARC)
+//   5. self-echo, part 2: our own recorded Message-ID
+//   5. route: short id, then legacy signed token, then the reply chain  (routes, does not authorize)
+//   6. strip quoted text, then filter attachments
+//   7. persist to the durable pending store  <- nothing is lost after this
+//   8. advance the cursor                     <- only after persist
+//
+// Marking \Seen and journaling happen later, in the daemon's /ack handler, once a
+// session has injected the reply. Doing it here would lose replies on a crash.
+
+import type { ResolvedConfig } from "../core/config.ts"
+import { authorize } from "../core/authz.ts"
+import { stripQuoted, type HeaderLike } from "../core/reply-parse.ts"
+import { tokenFromSubject } from "../core/token.ts"
+import { selectAttachments, type RawAttachment } from "../core/attachments.ts"
+import { findOutbound, parseMessageIds, lookupChain } from "../store/threads.ts"
+import { routeIdFromSubject, resolveRoute } from "../store/routes.ts"
+import { addPending, expirePending, listPending, type PendingRecord } from "../store/pending-store.ts"
+import { isJournaled } from "../store/journal.ts"
+import { advanceCursor, readCursor } from "../store/uid-cursor.ts"
+
+/** Longest body we keep. Well above any real instruction; far below "a whole newsletter". */
+export const MAX_BODY_CHARS = 100_000
+
+export interface RawMessage {
+	uid: number
+	from: string | undefined
+	subject: string | undefined
+	/** Plain-text body. */
+	text: string
+	/** Decoded attachments; the pipeline applies the allowlist and caps. */
+	attachments?: RawAttachment[]
+	headers: HeaderLike
+	receivedAt: number
+	/** Set when the server reported a message larger than we will parse. */
+	tooLarge?: number
+}
+
+/**
+ * Minimal mail source. `watcher.ts` implements this over imapflow; tests
+ * implement it over an in-memory fixture, so the pipeline never needs a socket.
+ */
+export interface MailSource {
+	/** The mailbox's UIDVALIDITY, or undefined when the source cannot report one. */
+	uidValidity(): number | undefined
+	/** UIDs strictly greater than `afterUid`, ascending. */
+	searchAfter(afterUid: number): Promise<number[]>
+	/** The message, or undefined when it vanished between search and fetch. */
+	fetch(uid: number): Promise<RawMessage | undefined>
+	/**
+	 * Mark the message seen on the server. Called only from `/ack`, i.e. after a
+	 * session has injected the reply — never at parse time.
+	 */
+	markSeen?(uid: number): Promise<void>
+}
+
+export interface ScanDeps {
+	source: MailSource
+	/** HMAC key for routing-token verification. */
+	secret: Buffer
+	env?: NodeJS.ProcessEnv
+	now?: () => number
+	/** Release a decision slot whose record was dropped, when nothing else holds it. */
+	onDropped?: (sessionId: string) => void
+}
+
+export interface ScanResult {
+	/** UIDs the source returned this scan. */
+	examined: number
+	/** Deliveries persisted to the pending store. */
+	persisted: number
+	/** Rejected: not for us, unauthorized, or machine-generated. */
+	rejected: number
+	/** Already journaled (acked earlier) — dedupe, not a rejection. */
+	skipped: number
+	/** Per-reason rejection counts, for logs and `/mailbridge status`. */
+	reasons: Record<string, number>
+	/** Cursor value after the scan. */
+	cursor: number
+}
+
+function bump(reasons: Record<string, number>, key: string): void {
+	reasons[key] = (reasons[key] ?? 0) + 1
+}
+
+/**
+ * One incremental scan.
+ *
+ * A message is "handled" (and the cursor moves past it) when it is persisted,
+ * rejected, skipped, or has vanished. Only a thrown error from the source or the
+ * pending store stops the scan with the cursor left before the offending UID, so
+ * the next scan retries it.
+ */
+export async function scanAndProcess(config: ResolvedConfig, deps: ScanDeps): Promise<ScanResult> {
+	const env = deps.env ?? process.env
+	const now = deps.now ?? (() => Date.now())
+
+	const startCursor = readCursor(env)
+	const uidValidity = deps.source.uidValidity?.()
+	const uids = (await deps.source.searchAfter(startCursor)).slice().sort((a, b) => a - b)
+
+	const result: ScanResult = {
+		examined: uids.length,
+		persisted: 0,
+		rejected: 0,
+		skipped: 0,
+		reasons: {},
+		cursor: startCursor,
+	}
+
+	for (const uid of uids) {
+		// Dedupe: an acked UID must never be re-offered, whatever the server says.
+		if (isJournaled(env, uid, uidValidity)) {
+			result.skipped++
+			bump(result.reasons, "journaled")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		const message = await deps.source.fetch(uid)
+		if (!message) {
+			// Deleted between search and fetch. Nothing to do; do not block the cursor.
+			result.skipped++
+			bump(result.reasons, "vanished")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		// The marker is stamped on every notification we send. Seeing it inbound means the
+		// mail is ours coming back - check before authorization, because in single-mailbox
+		// mode it is From: the human and would otherwise pass every gate.
+		if (message.headers.piMailbridge) {
+			result.rejected++
+			bump(result.reasons, "self-echo")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		// The watcher refuses to download anything over the size ceiling and marks it here.
+		// Reject before authorization: this is our resource guard, not the sender's fault.
+		if (message.tooLarge) {
+			result.rejected++
+			bump(result.reasons, "too-large")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		const authz = authorize({ from: message.from, headers: message.headers }, config)
+		if (!authz.ok) {
+			result.rejected++
+			bump(result.reasons, authz.reason)
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		// Self-echo, before routing: never treat our own notification as a reply. It arrives
+		// From: the mailbox itself, and because notifications now carry In-Reply-To (threading)
+		// the single-mailbox guard in authorize() can no longer tell them from a genuine
+		// reply. The outbound registry can: if this message's own Message-ID is one we sent,
+		// it is ours, whatever its headers say.
+		const ownIds = parseMessageIds(message.headers.messageId)
+		if (ownIds.length > 0 && ownIds.some((id) => findOutbound(env, id))) {
+			result.rejected++
+			bump(result.reasons, "self-echo")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		// Routing, in order of preference:
+		//   1. a short local route id  [pi:a1b2c3]  — what new notifications carry
+		//   2. a signed token          [pi:<base64>.<mac>] — messages already in flight
+		//   3. the reply chain         In-Reply-To / References → recorded Message-ID
+		// All three are routing only; authorization happened above.
+		let sessionId: string | undefined
+		const routeId = routeIdFromSubject(message.subject)
+
+		if (routeId) {
+			// Short ids are local, so an id minted on another machine - or one whose 30-day
+			// TTL has passed - does not resolve here. Fall through to the older rules rather
+			// than dropping the reply: a human answering an old notification should still
+			// reach the session through the token or the reply chain.
+			const route = resolveRoute(routeId, { device: config.device.name, now: now(), env })
+			if (route) {
+				sessionId = route.sessionId
+				bump(result.reasons, "routed")
+			} else {
+				bump(result.reasons, "route-miss")
+			}
+		}
+
+		if (!sessionId) {
+			const token = tokenFromSubject(message.subject, deps.secret, now(), config.device.name)
+			if (token) {
+				// deviceMatches === false means the token was minted for another machine.
+				// undefined means the format carries no device, which is not a mismatch.
+				if (token.deviceMatches === false || (token.device && token.device !== config.device.name)) {
+					result.rejected++
+					bump(result.reasons, "other-device")
+					result.cursor = advanceCursor(env, uid)
+					continue
+				}
+				sessionId = token.sessionId
+			} else if (config.security.allowHeaderThreading !== false) {
+				// The reply chain: a client or human that rewrote the subject still leaves
+				// In-Reply-To/References intact, which is how "reply to this mail" works.
+				const ids = [
+					...parseMessageIds(message.headers.inReplyTo),
+					...parseMessageIds((message.headers.references ?? []).join(" ")),
+				]
+				const thread = lookupChain(env, ids, { now: now(), ttlHours: config.security.tokenTtlHours })
+				if (thread?.device && thread.device !== config.device.name) {
+					result.rejected++
+					bump(result.reasons, "other-device")
+					result.cursor = advanceCursor(env, uid)
+					continue
+				}
+				if (thread) {
+					sessionId = thread.sessionId
+					bump(result.reasons, "threaded")
+				}
+			}
+		}
+
+		if (!sessionId) {
+			result.rejected++
+			bump(result.reasons, "no-token")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		// A body cap as well as a message cap: the pending store is re-read on every poll and
+		// on every /health, so one enormous reply would be paid for repeatedly.
+		const stripped = stripQuoted(message.text)
+		// Marked, not silently cut: the agent must be able to tell that its instruction is
+		// incomplete (the outbound truncation note does the same).
+		const body =
+			stripped.length > MAX_BODY_CHARS
+				? `${stripped.slice(0, MAX_BODY_CHARS)}\n\n[truncated: ${stripped.length - MAX_BODY_CHARS} more characters]`
+				: stripped
+
+		// Attachments are filtered BEFORE persisting; a dropped one is recorded with
+		// a reason so `/mailbridge` can explain why the agent did not see it.
+		const { accepted, dropped } = selectAttachments(message.attachments ?? [])
+		for (const item of dropped) bump(result.reasons, `attachment-${item.reason}`)
+
+		// A reply is empty only if it has neither prose nor a surviving attachment. Judging
+		// the body first discarded the image-from-a-phone case: nothing persisted, the
+		// attachment left unread, and the cursor moved past it.
+		//
+		// The same applies when the *only* content was an attachment that got dropped (a phone
+		// image over the 1 MB total, say). Rejecting it loses the human's whole reply with
+		// no trace, so it is persisted with a note naming the drop and delivered: the agent can
+		// ask for the text, and the mail is no longer sitting past the cursor unread.
+		if (!body && accepted.length === 0) {
+			const reasons = [...new Set(dropped.map((d) => d.reason))].join(", ")
+			if (reasons.length > 0) {
+				const note = `[pi-mailbridge: ${dropped.length} attachment(s) dropped (${reasons}); this reply had no text]`
+				// Not counted as rejected: the reply *is* delivered, as a note. The reason is
+				// named so the panel can say the content was lost, without claiming the mail was.
+				bump(result.reasons, "attachment-only-dropped")
+				const stub: PendingRecord = {
+					uid,
+					sessionId,
+					from: authz.from,
+					subject: message.subject ?? "",
+					body: note,
+					receivedAt: message.receivedAt ?? now(),
+					...(uidValidity === undefined ? {} : { uidValidity }),
+					device: config.device.name,
+				}
+				const persisted = addPending(env, stub)
+				if (!persisted.ok) {
+					// Same rule as a real record: do not advance past it, retry when there is room.
+					bump(result.reasons, "pending-full")
+					break
+				}
+				result.persisted++
+				result.cursor = advanceCursor(env, uid)
+				continue
+			}
+			result.rejected++
+			bump(result.reasons, "empty-body")
+			result.cursor = advanceCursor(env, uid)
+			continue
+		}
+
+		const record: PendingRecord = {
+			uid,
+			sessionId,
+			from: authz.from,
+			subject: message.subject ?? "",
+			body,
+			...(accepted.length ? { attachments: accepted } : {}),
+			receivedAt: message.receivedAt,
+			// Re-read per record: a renumber can land mid-scan, and stamping the scan's opening
+			// space would leave the tail of the batch unclaimable until the next connect.
+			...((deps.source.uidValidity?.() ?? uidValidity) === undefined
+				? {}
+				: { uidValidity: deps.source.uidValidity?.() ?? uidValidity }),
+		}
+
+		// Persist BEFORE the cursor moves. A crash here leaves the mail unseen.
+		const added = addPending(env, { ...record, device: config.device.name })
+		if (!added.ok) {
+			// The queue is full. Before stalling every future reply, drop records nothing can
+			// deliver any more - a session that has not come back in a week. This is the only
+			// place age-based expiry runs, because it is the only place the loss buys progress.
+			const expired = expirePending(env)
+			if (expired.length > 0) {
+				result.rejected++
+				bump(result.reasons, "expired")
+				// The same rule as the UID-space drop: free a slot only when nothing else is
+				// queued for that session, or a live record would lose its reservation.
+				const stillQueued = new Set(listPending(env).map((r) => r.sessionId))
+				for (const gone of expired) {
+					if (!stillQueued.has(gone.sessionId)) deps.onDropped?.(gone.sessionId)
+				}
+				const retry = addPending(env, { ...record, device: config.device.name })
+				if (retry.ok) {
+					result.persisted++
+					result.cursor = advanceCursor(env, uid)
+					continue
+				}
+			}
+			// The queue is full. STOP the batch: advancing past this UID would lose it, and
+			// so would processing a later UID (the cursor is monotonic, so it would jump
+			// over this one). The mail stays unread and the whole batch is retried once
+			// space appears.
+			result.rejected++
+			bump(result.reasons, "pending-full")
+			break
+		}
+		result.persisted++
+
+		result.cursor = advanceCursor(env, uid)
+	}
+
+	return result
+}

package/src/daemon/scan-queue.ts

@@ -0,0 +1,45 @@
+// One scan at a time, with at most one queued behind it.
+//
+// Three things ask for a scan concurrently: an IMAP `exists`, a reconnect's catch-up, and
+// mode turning on. Overlapping scans are not harmless - two that pass the journal check
+// before either persists would deliver the same reply twice, which is the one outcome this
+// design cannot undo. A burst of IDLE events also should not become a burst of full scans,
+// so a request that arrives while one is running coalesces into a single follow-up.
+
+export interface ScanQueue {
+	(why: string): Promise<void>
+	/** True while a scan is running or one is queued. */
+	busy(): boolean
+}
+
+export function createScanQueue(run: (why: string) => Promise<void>): ScanQueue {
+	let current: Promise<void> | undefined
+	let queued: string | undefined
+
+	const start = (why: string): Promise<void> => {
+		current = (async () => {
+			try {
+				await run(why)
+			} catch {
+				// A scan failure is logged by the caller; the queue must survive it.
+			} finally {
+				current = undefined
+				if (queued !== undefined) {
+					const next = queued
+					queued = undefined
+					void start(next)
+				}
+			}
+		})()
+		return current
+	}
+
+	const scan = (why: string): Promise<void> => {
+		if (!current) return start(why)
+		// Coalesce: one follow-up covers any number of requests that arrive mid-scan.
+		queued ??= why
+		return current
+	}
+
+	return Object.assign(scan, { busy: () => current !== undefined })
+}

package/src/daemon/script.ts

@@ -0,0 +1,45 @@
+// Which daemon entry should a session spawn?
+//
+// The pi manifest points at `src/extension.ts`, and pi loads it through jiti - so in a published
+// install the loaded module is TypeScript *inside node_modules*, where Node refuses type
+// stripping. The daemon beside it must therefore be the compiled `dist/` entry whenever one
+// exists, whichever shape this module has.
+//
+// Pure and exported so both layouts can be tested without installing a package.
+
+import { existsSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+
+/**
+ * Resolve the daemon script for the module at `moduleUrl`.
+ *
+ * - `dist/extension.js`     -> `dist/daemon/main.js`
+ * - `src/extension.ts`      -> `dist/daemon/main.js` when built (a published install)
+ * - `src/extension.ts` only -> `src/daemon/main.ts` (a checkout, run with the flag)
+ */
+export function resolveDaemonScript(moduleUrl: string | URL, exists: (path: string) => boolean = existsSync): string {
+	const here = fileURLToPath(moduleUrl)
+	const dir = dirname(here)
+	const root = dirname(dir)
+	const compiled = here.endsWith(".js")
+	const candidates = compiled
+		? [
+				// dist/extension.js -> dist/daemon/main.js
+				join(dir, "daemon", "main.js"),
+				join(root, "dist", "daemon", "main.js"),
+				join(dir, "daemon", "main.ts"),
+			]
+		: [
+				// src/extension.ts -> the build, when the package was published
+				join(root, "dist", "daemon", "main.js"),
+				join(dir, "daemon", "main.js"),
+				// a checkout without a build
+				join(dir, "daemon", "main.ts"),
+			]
+	for (const candidate of candidates) {
+		if (exists(candidate)) return candidate
+	}
+	// Nothing exists: name the source path, so the failure mentions what the caller expected.
+	return join(dir, "daemon", "main.ts")
+}

package/src/daemon/send-budget.ts

@@ -0,0 +1,36 @@
+// How much mail this daemon will send per minute.
+//
+// The delivery path is model-driven: `notify_user` and settle notifications can be produced by an
+// agent, and an injected agent can ask for a great many. Providers answer a burst by throttling
+// the account or refusing for the day, which silences the notifications that actually matter.
+// A small sliding window keeps a loop from spending the quota.
+//
+// Deliberately not a queue: a refused send is reported to the caller (`429 send_budget`), so the
+// agent sees it and the human is not mailed a backlog of stale notifications later.
+
+export interface SendBudget {
+	/** True when a send is allowed now; consumes one slot. */
+	take(now: number): boolean
+	/** Slots used inside the current window, for tests and diagnostics. */
+	used(now: number): number
+}
+
+export function createSendBudget(options: { limit: number; windowMs: number }): SendBudget {
+	const sent: number[] = []
+	const prune = (now: number): void => {
+		const floor = now - options.windowMs
+		while (sent.length > 0 && sent[0]! <= floor) sent.shift()
+	}
+	return {
+		take(now: number) {
+			prune(now)
+			if (sent.length >= options.limit) return false
+			sent.push(now)
+			return true
+		},
+		used(now: number) {
+			prune(now)
+			return sent.length
+		},
+	}
+}