@novacraft-engineering/mailbox 0.1.0

package/lib/turso.ts

@@ -0,0 +1,53 @@
+/**
+ * Turso (libSQL) over HTTP, exposing the same tagged-template shape the D1 client used
+ * so every call site in `mailbox.ts` reads unchanged. libSQL is SQLite, so the schema
+ * and its `ON CONFLICT` clauses port without translation.
+ *
+ * HTTP mode rather than WebSocket: Vercel functions are short-lived and a socket per
+ * invocation is churn we would only pay for.
+ */
+
+import { createClient, type Client, type InValue } from '@libsql/client'
+
+export type SqlTag = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<Record<string, unknown>[]>
+
+let cached: Client | null = null
+
+function client(): Client {
+  if (cached) return cached
+  const url = process.env.TURSO_DATABASE_URL
+  const authToken = process.env.TURSO_AUTH_TOKEN
+  if (!url) throw new Error('TURSO_DATABASE_URL must be configured')
+  // Embedded replicas and file: URLs need no token; a remote libsql:// URL does.
+  cached = createClient({ url, ...(authToken ? { authToken } : {}) })
+  return cached
+}
+
+/** SQLite has no boolean or object types — normalise before binding. */
+function toParam(value: unknown): InValue {
+  if (value === undefined || value === null) return null
+  if (typeof value === 'boolean') return value ? 1 : 0
+  if (typeof value === 'number' || typeof value === 'bigint') return value
+  if (value instanceof Date) return value.toISOString()
+  if (value instanceof Uint8Array) return value
+  if (typeof value === 'object') return JSON.stringify(value)
+  return String(value)
+}
+
+export async function tursoQuery(sql: string, params: unknown[] = []): Promise<Record<string, unknown>[]> {
+  const result = await client().execute({ sql, args: params.map(toParam) })
+  return result.rows as unknown as Record<string, unknown>[]
+}
+
+/** Tagged-template entry point: interpolations become bound `?` parameters. */
+export function turso(): SqlTag {
+  return (strings, ...values) => tursoQuery(strings.raw.join('?'), values)
+}
+
+/**
+ * Run several statements as one transaction. The D1 client could only loop, so a
+ * half-applied schema was possible; libSQL gives us the real thing.
+ */
+export async function tursoBatch(statements: string[]): Promise<void> {
+  await client().batch(statements.map(sql => ({ sql, args: [] })), 'write')
+}

package/next.config.ts

@@ -0,0 +1,51 @@
+import type { NextConfig } from "next";
+
+/**
+ * This is a private mailbox. Nothing here should ever be indexed, summarised or
+ * used as training data, so the refusal is sent as a header too — a crawler that
+ * ignores robots.txt still receives it on every response.
+ */
+const noIndexHeaders = [
+  { key: 'X-Robots-Tag', value: 'noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate' },
+  { key: 'X-Frame-Options', value: 'DENY' },
+  { key: 'X-Content-Type-Options', value: 'nosniff' },
+  { key: 'Referrer-Policy', value: 'no-referrer' },
+  { key: 'Permissions-Policy', value: 'camera=(), microphone=(), geolocation=(), interest-cohort=()' },
+  { key: 'Strict-Transport-Security', value: 'max-age=63072000; includeSubDomains; preload' },
+  { key: 'Cross-Origin-Opener-Policy', value: 'same-origin' },
+];
+
+const nextConfig: NextConfig = {
+  // Ensure the contact-form email templates ship with the serverless bundle.
+  // Without this, `readFileSync('lib/emails/templates/*.html')` works in dev
+  // but the .html files get tree-shaken out of the production trace.
+  outputFileTracingIncludes: {
+    '/api/contact': ['./lib/emails/templates/**/*.html'],
+  },
+  poweredByHeader: false,
+  env: {
+    // Captured at build time. On Vercel this is the last successful build's
+    // ISO timestamp; locally it's the time `next build` ran.
+    NEXT_PUBLIC_BUILD_TIME: new Date().toISOString(),
+  },
+  async headers() {
+    return [
+      { source: '/:path*', headers: noIndexHeaders },
+      // The viewer shows a PDF in an iframe pointed at our own route, and DENY refuses
+      // that as firmly as it refuses a stranger — the browser reports it as the site
+      // refusing to connect, which reads as the file being unreachable. These paths only
+      // ever return an attachment, and only to a signed-in caller, so they allow being
+      // framed by this origin and nothing else.
+      {
+        source: '/api/mail/inbox/attachments/download',
+        headers: [{ key: 'X-Frame-Options', value: 'SAMEORIGIN' }],
+      },
+      {
+        source: '/api/mail/emails/:id/attachments/download',
+        headers: [{ key: 'X-Frame-Options', value: 'SAMEORIGIN' }],
+      },
+    ];
+  },
+};
+
+export default nextConfig;

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@novacraft-engineering/mailbox",
+  "version": "0.1.0",
+  "description": "A shared webmail app: one codebase, one deployment per mailbox. Threads, a rich composer with signatures, attachments on S3-compatible storage, sharing links, full-text search, web push and PWA install.",
+  "keywords": [
+    "webmail",
+    "email",
+    "mailbox",
+    "nextjs",
+    "sqlite",
+    "self-hosted"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Jaybee4real/mailbox.git"
+  },
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "app",
+    "lib",
+    "public",
+    "scripts",
+    "tools",
+    "next.config.ts",
+    "tsconfig.json",
+    "eslint.config.mjs",
+    ".env.example",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "dev": "next dev -p $(perch port 2>/dev/null || echo 4030)",
+    "build": "next build",
+    "start": "next start -p 4030",
+    "lint": "eslint ."
+  },
+  "dependencies": {
+    "@libsql/client": "0.17.0",
+    "@tiptap/extension-color": "^3.31.3",
+    "@tiptap/extension-highlight": "^3.31.3",
+    "@tiptap/extension-image": "^3.31.3",
+    "@tiptap/extension-link": "^3.31.3",
+    "@tiptap/extension-table": "^3.31.3",
+    "@tiptap/extension-text-align": "^3.31.3",
+    "@tiptap/extension-text-style": "^3.31.3",
+    "@tiptap/extension-underline": "^3.31.3",
+    "@tiptap/pm": "^3.31.3",
+    "@tiptap/react": "^3.31.3",
+    "@tiptap/starter-kit": "^3.31.3",
+    "@vercel/blob": "^2.6.1",
+    "next": "16.2.6",
+    "react": "19.2.4",
+    "react-dom": "19.2.4",
+    "resend": "^6.12.3",
+    "web-push": "^3.6.7"
+  },
+  "devDependencies": {
+    "@types/node": "^22",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "@types/web-push": "^3.6.4",
+    "eslint": "^9.39.5",
+    "eslint-config-next": "^16.2.6",
+    "typescript": "^5"
+  }
+}

package/public/brand/README.md

@@ -0,0 +1,35 @@
+# Brand assets
+
+> **Never deploy or `git push` unless asked for it in that request.** Pushing
+> `master` auto-deploys every live mailbox. See the deploying section in the
+> root [README](../README.md).
+
+**No tenant's artwork belongs in this repository.** Each mailbox keeps its own
+images on its mounted volume, in the directory named by `BRAND_ASSET_DIR`
+(default `/data/brand`), and the app serves them from `/brand/<file>`.
+
+| File | Used by | Notes |
+|---|---|---|
+| `mark.png` | app chrome | Drawn through a CSS mask, so only the alpha channel matters. |
+| `mark-email.png` | email signatures | Must carry real colour — mail clients cannot apply a CSS mask. |
+| `logo.png` | share pages | Full lockup. |
+
+Point the configuration at them:
+
+```
+BRAND_MARK_URL=https://mail.example.com/brand/mark-email.png
+NEXT_PUBLIC_BRAND_MARK_URL=https://mail.example.com/brand/mark-email.png
+BRAND_CHROME_MARK_URL=/brand/mark.png
+NEXT_PUBLIC_BRAND_CHROME_MARK_URL=/brand/mark.png
+```
+
+Any of these may instead be a URL on a host the tenant already owns; nothing in
+the app requires the files to be served from here.
+
+## App icons
+
+`public/icon-192.png`, `public/icon-512.png`, `public/icon-maskable-512.png`,
+`app/icon.png` and `app/apple-icon.png` ship as a neutral placeholder — a plain
+envelope on the default accent. They are the favicon, the Apple touch icon and
+the icon a PWA install uses. `BRAND_ICON_URL` and `BRAND_APPLE_ICON_URL`
+override them per tenant without touching the repository.

package/public/sw.js

@@ -0,0 +1,41 @@
+self.addEventListener('install', () => self.skipWaiting())
+self.addEventListener('activate', event => event.waitUntil(self.clients.claim()))
+
+// Present but deliberately does not respond: installability needs a fetch handler, and a
+// mailbox must never be served from a cache. Every request falls through to the network.
+self.addEventListener('fetch', () => {})
+
+self.addEventListener('notificationclick', event => {
+  event.notification.close()
+  event.waitUntil(
+    self.clients.matchAll({ type: 'window', includeUncontrolled: true }).then(clients => {
+      const open = clients.find(client => client.url.includes('/mail'))
+      if (open) return open.focus()
+      return self.clients.openWindow(event.notification.data?.url || '/mail')
+    }),
+  )
+})
+
+// Web Push, sent by the inbound webhook for the mailbox this device registered.
+self.addEventListener('push', event => {
+  let payload = {}
+  try {
+    payload = event.data ? event.data.json() : {}
+  } catch {
+    payload = { title: 'New mail', body: event.data ? event.data.text() : '' }
+  }
+  event.waitUntil(
+    Promise.all([
+      self.registration.showNotification(payload.title || 'New mail', {
+        body: payload.body || '',
+        icon: '/icon-192.png',
+        badge: '/icon-192.png',
+        tag: payload.tag || 'mailbox',
+        data: { url: payload.url || '/mail' },
+      }),
+      self.clients
+        .matchAll({ type: 'window', includeUncontrolled: true })
+        .then(clients => clients.forEach(client => client.postMessage({ type: 'mail:new' }))),
+    ]),
+  )
+})

package/scripts/backfill-attachments.mjs

@@ -0,0 +1,96 @@
+/**
+ * Copies attachment bytes out of Resend and into our bucket, for messages received
+ * before the app started keeping its own copy.
+ *
+ * The work happens on the deployment, not here: this only drives it, so the files
+ * travel between the provider and the bucket inside one region instead of down to
+ * whatever connection is running the script.
+ *
+ * Resumable — a message is picked up only while it has no stored key, so stopping
+ * and re-running costs nothing.
+ *
+ *   MAINTENANCE_TOKEN=… node scripts/backfill-attachments.mjs [--limit 25] [--host https://…]
+ */
+
+const args = process.argv.slice(2)
+const flag = (name, fallback) => {
+  const at = args.indexOf(`--${name}`)
+  return at === -1 ? fallback : args[at + 1]
+}
+
+const HOST = flag('host', 'https://mail.example.com')
+const LIMIT = Number(flag('limit', 25))
+const TOKEN = process.env.MAINTENANCE_TOKEN
+
+if (!TOKEN) {
+  console.error('MAINTENANCE_TOKEN is not set.')
+  process.exit(1)
+}
+
+const readable = bytes => {
+  const units = ['B', 'KB', 'MB', 'GB']
+  let value = bytes
+  let unit = 0
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024
+    unit += 1
+  }
+  return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`
+}
+
+const call = async query => {
+  const response = await fetch(`${HOST}/api/mail/maintenance/attachments?${query}`, {
+    method: 'POST',
+    headers: { authorization: `Bearer ${TOKEN}` },
+  })
+  const text = await response.text()
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 200)}`)
+  return JSON.parse(text)
+}
+
+const started = Date.now()
+let copied = 0
+let files = 0
+let bytes = 0
+let failed = 0
+
+const first = await call(`limit=1&count=1`)
+const outstanding = first.remaining ?? 0
+copied += first.copied
+files += first.files
+bytes += first.bytes
+console.log(`  ${outstanding} message(s) still to copy\n`)
+
+for (let round = 1; outstanding > 0 && round <= 5000; round += 1) {
+  let result
+  try {
+    result = await call(`limit=${LIMIT}`)
+  } catch (err) {
+    console.log(`  round ${round}: ${err.message} — retrying in 10s`)
+    await new Promise(resolve => setTimeout(resolve, 10_000))
+    continue
+  }
+
+  if (result.scanned === 0) {
+    console.log('\n  nothing left to copy')
+    break
+  }
+
+  copied += result.copied
+  files += result.files
+  bytes += result.bytes
+  failed += result.failed
+
+  const elapsed = (Date.now() - started) / 1000
+  const rate = copied / Math.max(elapsed, 1)
+  const left = Math.max(outstanding - copied, 0)
+  const eta = rate > 0 ? Math.round(left / rate) : 0
+  process.stdout.write(
+    `\r  ${copied}/${outstanding} messages · ${files} files · ${readable(bytes)}` +
+      `${failed ? ` · ${failed} failed` : ''} · ~${Math.floor(eta / 60)}m ${eta % 60}s left    `,
+  )
+}
+
+const total = Math.round((Date.now() - started) / 1000)
+console.log(`\n\n  done in ${Math.floor(total / 60)}m ${total % 60}s`)
+console.log(`  ${copied} messages, ${files} files, ${readable(bytes)} copied${failed ? `, ${failed} failed` : ''}`)

package/scripts/backfill-content-ids.mjs

@@ -0,0 +1,101 @@
+/**
+ * Records the Content-ID of each stored file, which is what tells an embedded
+ * signature image apart from something the sender actually attached.
+ *
+ * Metadata only — the bytes are already in the bucket, so this moves nothing and
+ * costs one provider call per message.
+ *
+ *   node scripts/backfill-content-ids.mjs [--batch 40]
+ */
+
+import { createClient } from '@libsql/client'
+import { readFileSync } from 'node:fs'
+
+for (const line of readFileSync(new URL('../.env.local', import.meta.url), 'utf8').split('\n')) {
+  const match = line.match(/^([A-Z0-9_]+)=(.*)$/)
+  if (match && !process.env[match[1]]) process.env[match[1]] = match[2].replace(/^"|"$/g, '')
+}
+
+const args = process.argv.slice(2)
+const at = args.indexOf('--batch')
+const BATCH = at === -1 ? 40 : Number(args[at + 1])
+
+const db = createClient({ url: process.env.TURSO_DATABASE_URL, authToken: process.env.TURSO_AUTH_TOKEN })
+const apiKey = process.env.RESEND_API_KEY
+
+async function retry(work) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await work()
+    } catch (err) {
+      if (attempt >= 5) throw err
+      await new Promise(resolve => setTimeout(resolve, 3000 * attempt))
+    }
+  }
+}
+
+const PENDING = `FROM mail_inbox
+  WHERE id NOT LIKE 'mbox-%'
+    AND attachments LIKE '%"key"%'
+    AND attachments NOT LIKE '%contentId%'`
+
+const remaining = Number(
+  Object.values((await retry(() => db.execute(`SELECT COUNT(*) ${PENDING}`))).rows[0])[0],
+)
+console.log(`  ${remaining.toLocaleString()} message(s) to annotate\n`)
+
+let done = 0
+let embedded = 0
+const started = Date.now()
+
+for (let round = 1; round <= 5000; round += 1) {
+  const rows = (await retry(() => db.execute(`SELECT id, attachments ${PENDING} LIMIT ${BATCH}`))).rows
+  if (rows.length === 0) break
+
+  const updates = []
+  await Promise.all(
+    rows.map(async row => {
+      const id = String(row.id)
+      let stored = []
+      try {
+        stored = JSON.parse(String(row.attachments))
+      } catch {
+        return
+      }
+
+      const response = await fetch(`https://api.resend.com/emails/receiving/${encodeURIComponent(id)}/attachments`, {
+        headers: { authorization: `Bearer ${apiKey}` },
+      }).catch(() => null)
+
+      // Mark it either way: a message the provider no longer holds must not be
+      // re-selected forever, so it is annotated with what we already know.
+      const listed = response?.ok ? ((await response.json()).data ?? []) : []
+      const merged = stored.map((file, index) => {
+        const match = listed[index] ?? listed.find(entry => String(entry.filename ?? '') === String(file.filename ?? ''))
+        const raw = match?.content_id ? String(match.content_id).replace(/^<|>$/g, '') : ''
+        if (raw && String(file.contentType ?? '').toLowerCase().startsWith('image/')) embedded += 1
+        return { ...file, contentId: raw || null }
+      })
+
+      updates.push({ id, json: JSON.stringify(merged) })
+    }),
+  )
+
+  for (const update of updates) {
+    await retry(() =>
+      db.execute({
+        sql: 'UPDATE mail_inbox SET attachments = ?, attach_meta = ? WHERE id = ?',
+        args: [update.json, update.json, update.id],
+      }),
+    )
+  }
+  done += updates.length
+  if (updates.length === 0) break
+
+  const rate = done / Math.max((Date.now() - started) / 1000, 1)
+  process.stdout.write(
+    `\r  ${done}/${remaining} messages · ${embedded} embedded images found · ${rate.toFixed(1)}/s    `,
+  )
+}
+
+console.log(`\n\n  annotated ${done} message(s); ${embedded} embedded image(s) will now be hidden`)

package/scripts/backfill-list-columns.mjs

@@ -0,0 +1,80 @@
+/**
+ * Fills the columns the message list reads, so it stops falling back to the message
+ * body. The fallback reads body_text, which sits behind the html blob in the record,
+ * and that is the difference between a list query of ~200ms and one of ~2s.
+ *
+ * Newest first: those are the rows every mailbox opens on, so the first few thousand
+ * buy nearly all of the benefit long before the archive is done.
+ *
+ * Resumable — a row is picked up only while snippet is NULL.
+ *
+ *   node scripts/backfill-list-columns.mjs [--batch 500]
+ */
+
+import { createClient } from '@libsql/client'
+import { readFileSync } from 'node:fs'
+
+for (const line of readFileSync(new URL('../.env.local', import.meta.url), 'utf8').split('\n')) {
+  const match = line.match(/^([A-Z0-9_]+)=(.*)$/)
+  if (match && !process.env[match[1]]) process.env[match[1]] = match[2].replace(/^"|"$/g, '')
+}
+
+const args = process.argv.slice(2)
+const at = args.indexOf('--batch')
+const BATCH = at === -1 ? 500 : Number(args[at + 1])
+
+const db = createClient({
+  url: process.env.TURSO_DATABASE_URL,
+  authToken: process.env.TURSO_AUTH_TOKEN,
+})
+
+const UPDATE = `UPDATE mail_inbox SET
+   snippet = substr(COALESCE(body_text, ''), 1, 320),
+   thread_meta = json_object(
+     'message-id', COALESCE(json_extract(headers, '$."message-id"'), ''),
+     'in-reply-to', COALESCE(json_extract(headers, '$."in-reply-to"'), ''),
+     'references', COALESCE(json_extract(headers, '$."references"'), '')),
+   attach_meta = COALESCE(
+     (SELECT json_group_array(json_object('filename', json_extract(value, '$.filename'),
+                                          'contentType', json_extract(value, '$.contentType'),
+                                          'size', json_extract(value, '$.size')))
+      FROM json_each(CASE WHEN json_valid(attachments) THEN attachments ELSE '[]' END)),
+     '[]')
+ WHERE id IN (SELECT id FROM mail_inbox WHERE snippet IS NULL ORDER BY received_at DESC LIMIT ${BATCH})`
+
+/** Turso drops a connection now and then; a dropped batch is safe to repeat. */
+async function run(sql) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await db.execute(sql)
+    } catch (err) {
+      if (attempt >= 5) throw err
+      await new Promise(resolve => setTimeout(resolve, 3000 * attempt))
+    }
+  }
+}
+
+const outstanding = Number(Object.values((await run('SELECT COUNT(*) FROM mail_inbox WHERE snippet IS NULL')).rows[0])[0])
+console.log(`  ${outstanding.toLocaleString()} rows to fill, ${BATCH} at a time\n`)
+
+const started = Date.now()
+let done = 0
+
+while (done < outstanding) {
+  const result = await run(UPDATE)
+  if (!result.rowsAffected) break
+  done += result.rowsAffected
+
+  const elapsed = (Date.now() - started) / 1000
+  const rate = done / Math.max(elapsed, 1)
+  const eta = rate > 0 ? Math.round((outstanding - done) / rate) : 0
+  process.stdout.write(
+    `\r  ${done.toLocaleString()}/${outstanding.toLocaleString()} · ${rate.toFixed(0)}/s · ` +
+      `~${Math.floor(eta / 60)}m ${eta % 60}s left    `,
+  )
+}
+
+const total = Math.round((Date.now() - started) / 1000)
+const left = Number(Object.values((await run('SELECT COUNT(*) FROM mail_inbox WHERE snippet IS NULL')).rows[0])[0])
+console.log(`\n\n  filled ${done.toLocaleString()} rows in ${Math.floor(total / 60)}m ${total % 60}s`)
+console.log(`  still NULL: ${left.toLocaleString()}`)