@dumbmice/pi-mailbridge 0.0.0-stage → 0.1.1

package/src/store/routes.ts

@@ -0,0 +1,267 @@
+// Short routing ids.
+//
+// The signed token proved out but cost ~42 characters of base64 in every subject
+// line, which is most of what a mail client shows. Since the routing state is
+// already local (per-daemon cursor, journal, pending store, thread ids), the subject
+// only needs a local lookup key.
+//
+// So: mint a 6-hex-character id, remember which session it belongs to, and put
+// `[pi:a1b2c3]` in the subject. Twelve characters instead of forty-two.
+//
+// What this gives up and why it is acceptable:
+//   - The tag is not self-describing, so another machine cannot resolve it. That is
+//     already true of thread ids, and correct: a device must not act on another
+//     device's mail.
+//   - The tag is not signed. It does not need to be: it is only a lookup key, and an
+//     unknown key resolves to nothing. Authorization is still the sender allowlist
+//     plus DMARC (docs/SECURITY.md §2.2).
+//
+// Signed tokens keep working: messages minted before this change still route.
+
+import {
+	closeSync,
+	existsSync,
+	mkdirSync,
+	openSync,
+	readFileSync,
+	renameSync,
+	statSync,
+	unlinkSync,
+	writeFileSync,
+	writeSync,
+} from "node:fs"
+import { dirname } from "node:path"
+import { randomBytes } from "node:crypto"
+
+import { statePath } from "./paths.ts"
+import { StateVersionError, checkStateVersion, stamp } from "./schema.ts"
+
+/** Keep this many live routes. One per session (reused and refreshed), not per mail. */
+export const ROUTES_CAP = 2_000
+/** 6 hex characters: 16.7M values, ample for one mailbox's notifications. */
+const ID_BYTES = 3
+const ID_HEX = ID_BYTES * 2
+
+/** 6 hex normally; 8 is the rare collision fallback further down, and must still match. */
+export const ROUTE_RE = /\[\s*pi:([0-9a-f]{6,8})\s*\]/i
+
+export interface RouteRecord {
+	id: string
+	sessionId: string
+	device?: string
+	/** Epoch ms; the route stops resolving after this. */
+	expiresAt: number
+	subject: string
+}
+
+interface RoutesFile {
+	v?: number
+	items: RouteRecord[]
+}
+
+function path(env: NodeJS.ProcessEnv): string {
+	return statePath(env, "routes.json")
+}
+
+function load(env: NodeJS.ProcessEnv): RouteRecord[] {
+	const p = path(env)
+	try {
+		if (!existsSync(p)) return []
+		const parsed = JSON.parse(readFileSync(p, "utf8")) as Partial<RoutesFile>
+		checkStateVersion("routes.json", parsed)
+		if (!Array.isArray(parsed?.items)) return []
+		return parsed.items.filter(
+			(r): r is RouteRecord =>
+				typeof r?.id === "string" && typeof r?.sessionId === "string" && typeof r?.expiresAt === "number",
+		)
+	} catch (error) {
+		if (error instanceof StateVersionError) throw error
+		return []
+	}
+}
+
+/**
+ * Cross-process lock around the read-modify-write.
+ *
+ * routes.json is the one store written by every pi process, not just the daemon. Without a
+ * lock two sessions settling at once each write their whole in-memory copy and one route is
+ * lost - and a lost route means a reply falls back to the threading chain, or to `no-token`.
+ * The lock is an O_EXCL file, removed on release and stolen if a holder died.
+ */
+/**
+ * A lock older than this is treated as abandoned by a process that died holding it.
+ * Legitimate holds are single-digit milliseconds; this is only reached by a dead holder or a
+ * process stalled for seconds, which cannot be told apart without an OS-level flock.
+ */
+const LOCK_STALE_MS = 3_000
+/** Retry budget must exceed the stale threshold, or a stale lock can never be stolen. */
+const LOCK_ATTEMPTS = 900
+const LOCK_SLEEP_MS = 5
+
+function withLock<T>(env: NodeJS.ProcessEnv, run: () => T): T {
+	const lock = statePath(env, "routes.lock")
+	mkdirSync(dirname(lock), { recursive: true, mode: 0o700 })
+	// The owner is written into the file so a release only removes a lock it still holds.
+	const owner = `${process.pid}:${randomBytes(6).toString("hex")}`
+	const sleep = (): void => {
+		Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, LOCK_SLEEP_MS)
+	}
+	for (let attempt = 0; attempt < LOCK_ATTEMPTS; attempt++) {
+		try {
+			const fd = openSync(lock, "wx", 0o600)
+			try {
+				writeSync(fd, owner)
+			} catch {
+				/* the owner tag is a nicety; the lock itself is the guarantee */
+			}
+			try {
+				return run()
+			} finally {
+				closeSync(fd)
+				// Release by renaming the lock aside and checking the *renamed* file: rename is
+				// atomic, so a stealer that swapped the lock in between cannot have its own lock
+				// deleted by us (read-then-unlink had that window).
+				try {
+					const aside = `${lock}.mine.${process.pid}.${randomBytes(4).toString("hex")}`
+					renameSync(lock, aside)
+					if (readFileSync(aside, "utf8") === owner) unlinkSync(aside)
+					else renameSync(aside, lock) // we took somebody else's: put it back
+				} catch {
+					/* already gone, or the path was recreated: leave it alone */
+				}
+			}
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error
+			// Steal only a lock old enough to belong to a dead holder, and steal it by
+			// *renaming* it aside: rename is atomic, so two stealers cannot both succeed (the
+			// loser gets ENOENT). The previous unlink-based steal let a second stealer delete
+			// the lock the first had just created, putting both writers in the critical section.
+			try {
+				if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) {
+					const aside = `${lock}.stale.${process.pid}.${randomBytes(4).toString("hex")}`
+					renameSync(lock, aside)
+					unlinkSync(aside)
+				}
+			} catch {
+				/* raced with the holder or another stealer: retry */
+			}
+			sleep()
+		}
+	}
+	throw new Error("pi-mailbridge: routes.json is locked by another process")
+}
+
+function save(env: NodeJS.ProcessEnv, items: RouteRecord[]): void {
+	const p = path(env)
+	mkdirSync(dirname(p), { recursive: true, mode: 0o700 })
+	const tmp = `${p}.${process.pid}.tmp`
+	const trimmed = items.length > ROUTES_CAP ? items.slice(items.length - ROUTES_CAP) : items
+	writeFileSync(tmp, JSON.stringify({ ...stamp(), items: trimmed } satisfies RoutesFile), { mode: 0o600 })
+	renameSync(tmp, p)
+}
+
+export interface RegisterRouteOptions {
+	sessionId: string
+	subject: string
+	device?: string
+	ttlHours?: number
+	now?: number
+	env?: NodeJS.ProcessEnv
+}
+
+/** Mint a route for an outbound notification and return its short id. */
+export function registerRoute(options: RegisterRouteOptions): string {
+	const env = options.env ?? process.env
+	const now = options.now ?? Date.now()
+	const ttlHours = options.ttlHours ?? 30 * 24
+
+	return withLock(env, () => registerLocked(options, env, now, ttlHours))
+}
+
+function registerLocked(options: RegisterRouteOptions, env: NodeJS.ProcessEnv, now: number, ttlHours: number): string {
+	const items = load(env).filter((r) => r.expiresAt > now)
+	const taken = new Set(items.map((r) => r.id))
+
+	let id = ""
+	for (let attempt = 0; attempt < 32; attempt++) {
+		const candidate = randomBytes(ID_BYTES).toString("hex")
+		if (!taken.has(candidate)) {
+			id = candidate
+			break
+		}
+	}
+	// Astronomically unlikely; better to widen than to reuse someone's id.
+	if (!id) id = randomBytes(ID_BYTES + 1).toString("hex")
+
+	items.push({
+		id,
+		sessionId: options.sessionId,
+		...(options.device ? { device: options.device } : {}),
+		expiresAt: now + ttlHours * 3600_000,
+		subject: options.subject,
+	})
+	save(env, items)
+	return id
+}
+
+/**
+ * One route per **session**, reused and refreshed on every notification.
+ *
+ * Per-message ids made every subject unique, which defeats the conversation grouping
+ * that mail clients do by subject - each notification became its own thread. Keeping the
+ * id stable per session means the subject is byte-identical across notifications, so
+ * Outlook's conversation topic and Gmail's subject grouping both keep one thread, and
+ * the RFC 5322 headers we also set have something stable to point at.
+ */
+export function ensureRouteForSession(options: RegisterRouteOptions): string {
+	const env = options.env ?? process.env
+	const now = options.now ?? Date.now()
+	const ttlHours = options.ttlHours ?? 30 * 24
+
+	return withLock(env, () => {
+	const items = load(env)
+	const existing = items.find((r) => r.sessionId === options.sessionId && r.expiresAt > now)
+	if (existing) {
+		// Refresh so a long-lived session keeps its id, and keep the subject current.
+		existing.expiresAt = now + ttlHours * 3600_000
+		if (existing.device !== options.device) {
+			if (options.device) existing.device = options.device
+			else delete existing.device
+		}
+		existing.subject = options.subject
+		save(env, items)
+		return existing.id
+	}
+	// registerLocked, not registerRoute: the lock is not re-entrant.
+	return registerLocked(options, env, now, ttlHours)
+	})
+}
+
+export interface ResolveRouteOptions {
+	/** The scanning device; a route minted elsewhere does not resolve here. */
+	device?: string
+	now?: number
+	env?: NodeJS.ProcessEnv
+}
+
+/** Look a short id up. Undefined when unknown, expired, or minted elsewhere. */
+export function resolveRoute(id: string, options: ResolveRouteOptions = {}): RouteRecord | undefined {
+	const env = options.env ?? process.env
+	const now = options.now ?? Date.now()
+	const normalized = id.trim().toLowerCase()
+	const record = load(env).find((r) => r.id === normalized)
+	if (!record) return undefined
+	if (record.expiresAt <= now) return undefined
+	if (record.device && options.device && record.device !== options.device) return undefined
+	return record
+}
+
+/** Extract the short id from a subject line, if present. */
+export function routeIdFromSubject(subject: string | undefined | null): string | undefined {
+	if (!subject) return undefined
+	return ROUTE_RE.exec(subject)?.[1]?.toLowerCase()
+}
+
+export function routeCount(env: NodeJS.ProcessEnv = process.env): number {
+	return load(env).length
+}

package/src/store/schema.ts

@@ -0,0 +1,104 @@
+// On-disk schema versions.
+//
+// Separate from API_VERSION (daemon/api.ts) on purpose: that one asks "can these two processes
+// talk", and is recovered by replacing a process. This one asks "can this build read the bytes",
+// and the data outlives every process - a downgrade, a shared or synced state directory, or two
+// builds pointed at one XDG_STATE_HOME.
+//
+// The policy every store follows (AGENTS.md "Compatibility"):
+//
+//   absent  -> schema 1: written before the field existed; read it, upgrade on the next save.
+//   older   -> migrate on read, write the current version on the next save.
+//   current -> parse normally.
+//   newer   -> refuse to read AND refuse to write, naming both versions. Filtering unknown
+//              records and saving the rest is how state is silently lost.
+
+import { readFileSync } from "node:fs"
+
+import { statePath } from "./paths.ts"
+
+export const STATE_VERSION = 1
+
+/**
+ * The version assumed for a file written before the field existed.
+ *
+ * Deliberately a constant, not `STATE_VERSION`: if this read `STATE_VERSION`, then bumping the
+ * current version would silently label a legacy file as current and skip its migration.
+ */
+export const LEGACY_SCHEMA = 1
+
+/** Every versioned file, so one probe can refuse the whole directory up front. */
+export const VERSIONED_FILES = [
+	"pending.json",
+	"journal.json",
+	"routes.json",
+	"threads.json",
+	"uid-cursor.json",
+] as const
+
+export class StateVersionError extends Error {
+	readonly file: string
+	readonly found: unknown
+	readonly current: number
+
+	constructor(file: string, found: unknown) {
+		super(
+			`${file} was written by a newer version of pi-mailbridge (schema ${String(found)}; ` +
+				`this build understands ${STATE_VERSION}). Refusing to read or rewrite it - update ` +
+				`pi-mailbridge, or move that file aside to start fresh.`,
+		)
+		this.name = "StateVersionError"
+		this.file = file
+		this.found = found
+		this.current = STATE_VERSION
+	}
+}
+
+/**
+ * Validate the version of an already-parsed file.
+ *
+ * Returns the version to interpret the payload as. Throws `StateVersionError` for anything from
+ * the future, which is the only case where reading would corrupt: an older shape is migrated.
+ */
+export function checkStateVersion(file: string, parsed: unknown): number {
+	const version = (parsed as { v?: unknown } | null | undefined)?.v
+	if (version === undefined) return LEGACY_SCHEMA
+	if (typeof version !== "number" || !Number.isFinite(version) || version < 0) {
+		throw new StateVersionError(file, version)
+	}
+	if (version > STATE_VERSION) throw new StateVersionError(file, version)
+	if (version < STATE_VERSION) {
+		// No migrations are registered yet. This guard makes a version bump fail loudly here
+		// instead of quietly reading pre-bump data with post-bump assumptions.
+		throw new Error(
+			`${file} is schema ${version}; this build is ${STATE_VERSION} and has no migration for it. ` +
+				`Register one in store/schema.ts before bumping STATE_VERSION.`,
+		)
+	}
+	return version
+}
+
+/** The version to write, stamped by every save. */
+export function stamp(): { v: number } {
+	return { v: STATE_VERSION }
+}
+
+/**
+ * Read every versioned file's `v` and refuse the directory if any is newer.
+ *
+ * Called once at boot, before the daemon listens: a load-time throw inside a request would
+ * surface as a 500 and inside a scan as a repeating log line, while this is one clear message.
+ * Missing or unparsable files are left to their stores (they have their own recovery), so this
+ * only fails on a *version* the build does not understand.
+ */
+export function assertStateReadable(env: NodeJS.ProcessEnv): void {
+	for (const name of VERSIONED_FILES) {
+		let parsed: unknown
+		try {
+			parsed = JSON.parse(readFileSync(statePath(env, name), "utf8"))
+		} catch {
+			continue
+		}
+		checkStateVersion(name, parsed)
+	}
+}

package/src/store/secret.ts

@@ -0,0 +1,69 @@
+// Shared secret material: one 32-byte key used both as the HMAC key for routing
+// tokens and as the daemon's bearer token.
+//
+// Created first-writer-wins with mode 0600 so the plugin can mint a token before
+// the daemon has ever started (the first mail must not fail because the daemon
+// is down).
+
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, writeSync, chmodSync } from "node:fs"
+
+import { stateDir, statePath } from "./paths.ts"
+import { newSecret } from "../core/token.ts"
+
+function ensureDir(env: NodeJS.ProcessEnv): string {
+	const dir = stateDir(env)
+	mkdirSync(dir, { recursive: true, mode: 0o700 })
+	return dir
+}
+
+/** Read the key, or undefined when it does not exist yet. Never creates. */
+export function readSecret(env: NodeJS.ProcessEnv = process.env): Buffer | undefined {
+	const path = statePath(env, "secret.key")
+	try {
+		if (!existsSync(path)) return undefined
+		const raw = readFileSync(path, "utf8").trim()
+		if (!raw) return undefined
+		const buf = Buffer.from(raw, "base64url")
+		return buf.length === 32 ? buf : undefined
+	} catch {
+		return undefined
+	}
+}
+
+/**
+ * Read the key, creating it if absent. Atomic: `wx` create means exactly one
+ * process wins and the others read the winner's key.
+ */
+/** Small synchronous pause; no timer is available on this path. */
+function sleepMs(ms: number): void {
+	Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
+}
+
+export function ensureSecret(env: NodeJS.ProcessEnv = process.env): Buffer {
+	const existing = readSecret(env)
+	if (existing) return existing
+
+	ensureDir(env)
+	const path = statePath(env, "secret.key")
+	try {
+		const fd = openSync(path, "wx", 0o600)
+		try {
+			writeSync(fd, newSecret().toString("base64url"))
+		} finally {
+			closeSync(fd)
+		}
+		chmodSync(path, 0o600)
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error
+		// Lost the race: the winner's key is the truth.
+	}
+	// Another process may have won the create race a microsecond ago and not finished
+	// writing: `openSync(...,"wx")` makes the file visible before writeSync lands. Retry
+	// briefly before declaring the key corrupt.
+	for (let attempt = 0; attempt < 50; attempt++) {
+		const winner = readSecret(env)
+		if (winner) return winner
+		sleepMs(2)
+	}
+	throw new Error(`pi-mailbridge: ${path} exists but is not a valid 32-byte key`)
+}

package/src/store/threads.ts

@@ -0,0 +1,177 @@
+// Outbound Message-ID registry — the reply-chain fallback.
+//
+// The short route id in the subject is the primary way a reply is identified, but a human
+// can edit a subject and some clients rewrite it. Mail clients, however, keep
+// `In-Reply-To` / `References` intact: that is how threading works, and it is what
+// "reply to this mail" actually does.
+//
+// So every outbound notification's Message-ID is recorded here, mapped to the
+// session (and device) it came from. When a reply arrives with no usable token,
+// the scan walks its References chain and accepts the first id we minted.
+//
+// Threat note: a Message-ID is unguessable in practice — it is only visible to the
+// recipient of the mail — and this path still sits behind the sender allowlist and
+// DMARC. It is a convenience fallback, not an authentication mechanism.
+//
+// Capped and atomic, like every other store here.
+
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"
+import { dirname } from "node:path"
+
+import { statePath } from "./paths.ts"
+import { StateVersionError, checkStateVersion, stamp } from "./schema.ts"
+
+/** Keep this many recent outbound ids. Replies to older mail fall back to the token. */
+export const THREAD_CAP = 2_000
+
+/** Matches the token default, so all three routing rules age out together. */
+export const DEFAULT_CHAIN_TTL_HOURS = 30 * 24
+
+export interface ThreadRecord {
+	/** Normalized Message-ID without angle brackets. */
+	messageId: string
+	sessionId: string
+	device?: string
+	subject: string
+	createdAt: number
+}
+
+interface ThreadFile {
+	v?: number
+	items: ThreadRecord[]
+}
+
+/** Strip angle brackets and surrounding whitespace: `<a@b>` -> `a@b`. */
+export function normalizeMessageId(raw: string | undefined | null): string {
+	if (!raw) return ""
+	return raw.trim().replace(/^<+/, "").replace(/>+$/, "").trim()
+}
+
+/** Every Message-ID inside a References/In-Reply-To header value. */
+export function parseMessageIds(raw: string | undefined | null): string[] {
+	if (!raw) return []
+	const matches = raw.match(/<[^<>]+>/g)
+	if (matches) return matches.map((m) => normalizeMessageId(m)).filter(Boolean)
+	// Tolerate clients that drop the angle brackets.
+	return raw
+		.split(/\s+/)
+		.map((part) => normalizeMessageId(part))
+		.filter((part) => part.includes("@"))
+}
+
+function path(env: NodeJS.ProcessEnv): string {
+	return statePath(env, "threads.json")
+}
+
+function load(env: NodeJS.ProcessEnv): ThreadRecord[] {
+	const p = path(env)
+	try {
+		if (!existsSync(p)) return []
+		const parsed = JSON.parse(readFileSync(p, "utf8")) as Partial<ThreadFile>
+		checkStateVersion("threads.json", parsed)
+		if (!Array.isArray(parsed?.items)) return []
+		return parsed.items.filter(
+			(r): r is ThreadRecord =>
+				typeof r?.messageId === "string" && typeof r?.sessionId === "string" && Boolean(r.messageId),
+		)
+	} catch (error) {
+		if (error instanceof StateVersionError) throw error
+		return []
+	}
+}
+
+function save(env: NodeJS.ProcessEnv, items: ThreadRecord[]): void {
+	const p = path(env)
+	mkdirSync(dirname(p), { recursive: true, mode: 0o700 })
+	const tmp = `${p}.${process.pid}.tmp`
+	const trimmed = items.length > THREAD_CAP ? items.slice(items.length - THREAD_CAP) : items
+	writeFileSync(tmp, JSON.stringify({ ...stamp(), items: trimmed } satisfies ThreadFile), { mode: 0o600 })
+	renameSync(tmp, p)
+}
+
+/** Remember an outbound notification so its replies can be traced back. */
+export function recordOutbound(
+	env: NodeJS.ProcessEnv,
+	record: { messageId: string; sessionId: string; device?: string; subject: string; createdAt?: number },
+): ThreadRecord | undefined {
+	const messageId = normalizeMessageId(record.messageId)
+	if (!messageId || !record.sessionId) return undefined
+
+	const entry: ThreadRecord = {
+		messageId,
+		sessionId: record.sessionId,
+		...(record.device ? { device: record.device } : {}),
+		subject: record.subject,
+		createdAt: record.createdAt ?? Date.now(),
+	}
+
+	const items = load(env).filter((r) => r.messageId !== messageId)
+	items.push(entry)
+	save(env, items)
+	return entry
+}
+
+/**
+ * First recorded outbound message among `ids`, searched in order.
+ *
+ * References lists ancestors oldest-first, so we search from the newest backward:
+ * the closest ancestor is the notification the human actually replied to.
+ */
+export function lookupChain(
+	env: NodeJS.ProcessEnv,
+	ids: readonly string[],
+	opts: { now?: number; ttlHours?: number } = {},
+): ThreadRecord | undefined {
+	if (ids.length === 0) return undefined
+	const now = opts.now ?? Date.now()
+	// The chain is the last routing rule, and it must not outlive the bound the other rules
+	// carry: without this, a re-sent copy of a month-old notification still routes into the
+	// session after route and token expiry have both passed, which made tokenTtlHours
+	// decorative for the ordinary reply case.
+	const oldest = now - (opts.ttlHours ?? DEFAULT_CHAIN_TTL_HOURS) * 3_600_000
+	const byId = new Map(load(env).filter((r) => r.createdAt >= oldest).map((r) => [r.messageId, r]))
+	for (let i = ids.length - 1; i >= 0; i--) {
+		const hit = byId.get(normalizeMessageId(ids[i]!))
+		if (hit) return hit
+	}
+	return undefined
+}
+
+/**
+ * Is this id one we sent? Age is irrelevant here: an old notification of ours is still ours,
+ * and recognising it is what stops it being re-ingested as a reply.
+ */
+export function findOutbound(env: NodeJS.ProcessEnv, id: string): ThreadRecord | undefined {
+	const target = normalizeMessageId(id)
+	return load(env).find((r) => r.messageId === target)
+}
+
+/**
+ * Forget one outbound id. Used when a send fails after the id was recorded ahead of the
+ * transport: leaving it would make the next notification reply to a parent that never
+ * existed, and would make a re-sent copy look like our own mail.
+ */
+export function removeOutbound(env: NodeJS.ProcessEnv, messageId: string): void {
+	const target = normalizeMessageId(messageId)
+	const items = load(env).filter((r) => r.messageId !== target)
+	save(env, items)
+}
+
+export function threadCount(env: NodeJS.ProcessEnv): number {
+	return load(env).length
+}
+
+/**
+ * The message ids for one session, oldest first use: `root` is what every later message
+ * references, `last` is what the next one replies to. Together they let a mail client
+ * keep all notifications for a session in one conversation.
+ */
+export function sessionThread(
+	env: NodeJS.ProcessEnv,
+	sessionId: string,
+): { rootMessageId?: string; lastMessageId?: string } {
+	if (!sessionId) return {}
+	const mine = load(env).filter((r) => r.sessionId === sessionId)
+	if (mine.length === 0) return {}
+	return { rootMessageId: mine[0]!.messageId, lastMessageId: mine[mine.length - 1]!.messageId }
+}