@novacraft-engineering/mailbox 0.2.0 → 0.3.0

package/app/mail/quoted.ts

@@ -0,0 +1,162 @@
+/**
+ * Where a received message stops being new and starts repeating the thread.
+ *
+ * Mail clients append the entire earlier conversation under a reply, so a long thread
+ * carries the first message a dozen times over. The boundary is found, the repeat is
+ * folded away, and the reader is offered it rather than made to scroll past it.
+ *
+ * Every rule here was measured against the real mailbox, and the near misses matter more
+ * than the hits: a bare "From:" line fires on ordinary sentences a thousand times over, and
+ * a row of underscores is a decoration as often as a divider. Only the conjunctions are
+ * trusted — the divider AND the header block that must follow it.
+ */
+
+const CONTAINER = [
+  // Outlook stacks a prefix onto the id on every requote, so match the suffix.
+  '[id$="divRplyFwdMsg" i]',
+  // Outlook Web draws this rule immediately above that div; every instance in the corpus
+  // sits on a real boundary, while a bare <hr> is decorative half the time.
+  'hr[style*="inline-block"][style*="98%"]',
+  // Outlook desktop flattens the same boundary to a bordered div.
+  'div[style*="border-top"][style*="#E1E1E1" i]',
+  '[class*="gmail_quote_container" i]',
+  'blockquote[class*="gmail_quote" i]',
+  'blockquote[type="cite" i]',
+  'blockquote[id*="blockquote_zmail" i]',
+  '[class*="moz-cite-prefix" i]',
+].join(',')
+
+// Deliberately absent: Outlook's empty "appendonsend" anchor. It marks where typing stopped,
+// which looks like the ideal cut, but the sender's signature is sometimes emitted after it.
+
+/** A reply divider: everything below it is the thread being repeated. */
+const REPLY_DIVIDER = [
+  /^\s*On\b[\s\S]{10,220}?\bwrote:\s*$/i,
+  /^\s*-{4,}\s*On\b[\s\S]{10,220}?\bwrote\s*-{4,}\s*$/i,
+]
+
+/**
+ * A forward marker means the opposite of a reply divider: what follows is not a repeat,
+ * it is the whole point of the message. Folding it on a bare forward would leave the
+ * reader a cover note and a fold line, so it is only honoured under a real covering note.
+ */
+const FORWARD_MARKER = [
+  /^\s*-{2,20}\s*(?:Original Message|Original message|Forwarded message)\s*-{2,20}\s*$/i,
+  /^\s*Begin forwarded message:\s*$/i,
+]
+
+/** A header block — only trusted when From: is followed by Sent:/Date:, never alone. */
+const HEADER_BLOCK = /^\s*(?:\*\s*)?(?:From|De)\s*:\s*\S[\s\S]{0,400}?^\s*(?:\*\s*)?(?:Sent|Date|Enviado)\s*:/im
+
+/** Text the reader must keep: if trimming leaves less than this, nothing is trimmed. */
+const MIN_HEAD_CHARS = 25
+/** A forward needs a substantial note of its own before its body may be folded. */
+const MIN_HEAD_CHARS_FORWARD = 150
+
+type Boundary = 'reply' | 'forward' | null
+
+function quoteStart(element: Element): Boundary {
+  if (element.matches(CONTAINER)) return 'reply'
+  const text = (element.textContent ?? '').trim()
+  if (!text) return null
+  if (text.length < 400 && REPLY_DIVIDER.some(pattern => pattern.test(text))) return 'reply'
+  if (text.length < 400 && FORWARD_MARKER.some(pattern => pattern.test(text))) return 'forward'
+  // A divider drawn as a rule or a run of underscores only counts when a header block follows.
+  const looksLikeDivider = element.tagName === 'HR' || /^[_—-]{10,}$/.test(text)
+  if (looksLikeDivider) return null
+  return text.length < 1200 && HEADER_BLOCK.test(text) ? 'reply' : null
+}
+
+export type QuotedSplit = { head: string; tail: string | null }
+
+/**
+ * Splits a message body into what was written now and what is being repeated.
+ * Returns tail = null when there is no repeat, or when trimming would leave nothing to read.
+ */
+export function splitQuotedTail(html: string): QuotedSplit {
+  if (typeof window === 'undefined' || !html) return { head: html, tail: null }
+  let doc: Document
+  try {
+    doc = new DOMParser().parseFromString(html, 'text/html')
+  } catch {
+    return { head: html, tail: null }
+  }
+  const body = doc.body
+  if (!body) return { head: html, tail: null }
+
+  // Walk the outermost run of siblings, descending through wrapper divs that hold everything.
+  let scope: Element = body
+  for (let depth = 0; depth < 6; depth++) {
+    const elements = Array.from(scope.children)
+    if (elements.length !== 1 || !/^(div|table|tbody|tr|td|section|article)$/i.test(elements[0].tagName)) break
+    if (elements[0].matches(CONTAINER)) break
+    scope = elements[0]
+  }
+
+  const children = Array.from(scope.children)
+  let at = -1
+  let kind: Boundary = null
+  for (const [index, element] of children.entries()) {
+    const boundary = quoteStart(element)
+    if (boundary) { at = index; kind = boundary; break }
+  }
+  if (at <= 0) return { head: html, tail: null }
+
+  const head = children.slice(0, at)
+  const kept = head.map(element => element.textContent ?? '').join(' ').trim().length
+  if (kept < (kind === 'forward' ? MIN_HEAD_CHARS_FORWARD : MIN_HEAD_CHARS)) {
+    return { head: html, tail: null }
+  }
+  const tail = children.slice(at)
+  return {
+    head: head.map(element => element.outerHTML).join(''),
+    tail: tail.map(element => element.outerHTML).join(''),
+  }
+}
+
+/**
+ * The same boundary in a message that arrived as plain text only.
+ *
+ * Half the bodies in the archive are text with no HTML part, and they quote just as
+ * heavily — so the reader needs this as much as the HTML path. Each pattern below was
+ * counted against the real archive, and the two that looked obvious are the two that
+ * had to be rejected: a lone "From:" heads every bounce report, and a row of underscores
+ * is a confidentiality footer more often than a divider.
+ */
+const TEXT_REPLY = [
+  // A run of underscores, but only where the quoted headers actually follow it.
+  /(?:^|\r?\n)[ \t]*_{10,}[ \t]*(?:\r?\n)+(?=[ \t]*(?:>[ \t]*)*(?:From|De)[ \t]*:[ \t])/,
+  // The header block itself: From, then Sent or Date, then To or Cc.
+  /(?:^|\r?\n)(?:>[ \t]*)*(?:From|De)[ \t]*:[ \t][^\n]{1,400}(?:[ \t]*\r?\n)+(?:>[ \t]*)*(?:Sent|Date|Enviado)[ \t]*:[ \t][^\n]{1,200}(?:[ \t]*\r?\n)+(?:>[ \t]*)*(?:To|Cc|Para)[ \t]*:[ \t]/,
+  // The attribution sentence, allowed to wrap over a few lines.
+  /(?:^|\r?\n)[ \t>]*On\b(?:[^\n]*\n){0,3}?[^\n]*\bwrote:[ \t]*(?=\r?\n|$)/,
+  /(?:^|\r?\n)[ \t>]*-{4,}[ \t]*On\b[^\n]{0,300}?wrote[ \t]*-{4,}[ \t]*$/m,
+]
+
+const TEXT_FORWARD = [
+  /(?:^|\r?\n)[ \t>]*-{2,20}[ \t]?(?:Original Message|Original message|Forwarded message)[ \t]?-{2,20}[ \t]*(?=\r?\n|$)/,
+  /(?:^|\r?\n)[ \t]*Begin forwarded message:[ \t]*(?=\r?\n|$)/,
+]
+
+export type TextSplit = { head: string; tail: string | null }
+
+export function splitQuotedText(text: string): TextSplit {
+  if (!text) return { head: text, tail: null }
+  let cut = -1
+  let kind: Boundary = null
+  for (const [patterns, boundary] of [[TEXT_REPLY, 'reply'], [TEXT_FORWARD, 'forward']] as const) {
+    for (const pattern of patterns) {
+      const match = pattern.exec(text)
+      if (!match) continue
+      // Cut at the divider itself, not after the newline that introduced it.
+      const at = match.index + (/^\r?\n/.test(match[0]) ? 1 : 0)
+      if (cut === -1 || at < cut) { cut = at; kind = boundary }
+    }
+  }
+  if (cut <= 0) return { head: text, tail: null }
+  const head = text.slice(0, cut)
+  if (head.trim().length < (kind === 'forward' ? MIN_HEAD_CHARS_FORWARD : MIN_HEAD_CHARS)) {
+    return { head: text, tail: null }
+  }
+  return { head: head.replace(/\s+$/, ''), tail: text.slice(cut) }
+}

package/app/mail/search.ts

@@ -164,6 +164,56 @@ function matchTerm(term: Term, doc: SearchDoc): boolean {
   }
 }
 
+export type ServerSearch = {
+  text: string
+  from?: string
+  to?: string
+  label?: string
+  unread?: boolean
+  starred?: boolean
+  hasAttachment?: boolean
+}
+
+/**
+ * The parts of a query the inbox route can answer itself go to the server; whatever it
+ * cannot express stays behind as the residual the browser still has to check.
+ */
+export function splitForServer(query: ParsedQuery): { server: ServerSearch; residual: ParsedQuery } {
+  const server: ServerSearch = { text: '' }
+  const words: string[] = []
+  const residual: Term[][] = []
+  for (const group of query.groups) {
+    const term = group.length === 1 ? group[0] : null
+    if (!term || term.negated) {
+      residual.push(group)
+      continue
+    }
+    if (term.field === null) words.push(term.value)
+    else if (term.field === 'from' && !server.from) server.from = term.value
+    else if (term.field === 'to' && !server.to) server.to = term.value
+    else if (term.field === 'label' && !server.label) server.label = term.value
+    else if (term.field === 'is' && term.value === 'unread') server.unread = true
+    else if (term.field === 'is' && term.value === 'starred') server.starred = true
+    else if (term.field === 'has' && term.value === 'attachment') server.hasAttachment = true
+    else residual.push(group)
+  }
+  server.text = words.join(' ')
+  return { server, residual: { groups: residual, isEmpty: residual.length === 0 } }
+}
+
+export function serverSearchParams(raw: string): Record<string, string> {
+  const { server } = splitForServer(parseQuery(raw))
+  const params: Record<string, string> = {}
+  if (server.text) params.q = server.text
+  if (server.from) params.from = server.from
+  if (server.to) params.to = server.to
+  if (server.label) params.label = server.label
+  if (server.unread) params.unread = '1'
+  if (server.starred) params.starred = '1'
+  if (server.hasAttachment) params.attachment = '1'
+  return params
+}
+
 export function matchesQuery(query: ParsedQuery, doc: SearchDoc): boolean {
   if (query.isEmpty) return true
   return query.groups.every(group =>

package/lib/mailbox.ts

@@ -1551,6 +1551,8 @@ export type SentMessage = {
   lastEvent?: string | null
   /** The bucket keys that went out with it, so a forward has something of ours to copy. */
   attachments?: Array<{ filename: string; size?: number; contentType?: string; key?: string }>
+  /** How many files the list should raise a paperclip for, counted the way the inbox counts. */
+  attachmentCount?: number
 }
 
 export async function recordSentMessage(message: SentMessage): Promise<void> {
@@ -1672,6 +1674,7 @@ export async function readSentArchive(options: {
   const rows = await tagged(
     sql,
     `SELECT s.id, s.from_addr, s.to_addrs, s.cc, s.bcc, s.reply_to, s.subject, s.created_at, s.last_event,
+            json_array_length(CASE WHEN json_valid(s.attachments) THEN s.attachments ELSE '[]' END) AS attach_count,
             ${textColumn} AS text
      FROM mail_sent s LEFT JOIN mail_sent_meta m ON m.email_id = s.id
      ${where.length ? `WHERE ${where.join(' AND ')}` : ''}
@@ -1690,6 +1693,7 @@ export async function readSentArchive(options: {
     text: row.text == null ? null : String(row.text),
     createdAt: isoOrNull(row.created_at) ?? new Date(0).toISOString(),
     lastEvent: (row.last_event as string) ?? null,
+    attachmentCount: Number(row.attach_count ?? 0),
   }))
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@novacraft-engineering/mailbox",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "A shared webmail app: one codebase, one deployment per mailbox. Threads, a rich composer with signatures, attachments on S3-compatible storage, sharing links, full-text search, web push and PWA install.",
   "keywords": [
     "webmail",
@@ -50,7 +50,6 @@
     "@tiptap/pm": "^3.31.3",
     "@tiptap/react": "^3.31.3",
     "@tiptap/starter-kit": "^3.31.3",
-    "@vercel/blob": "^2.6.1",
     "next": "^16.3.5",
     "react": "19.2.4",
     "react-dom": "19.2.4",

package/tools/import-mbox.py

@@ -23,6 +23,7 @@ import os
 import re
 import socket
 import sys
+import sqlite3
 import threading
 import time
 import urllib.error
@@ -73,7 +74,7 @@ INBOX_COLUMNS = ('id, from_addr, to_addrs, cc, bcc, reply_to, subject, html, bod
 INSERT_INBOX = (f'INSERT INTO mail_inbox ({INBOX_COLUMNS}) VALUES ({",".join("?" * 18)}) '
                 'ON CONFLICT (id) DO NOTHING')
 INSERT_SENT = ('INSERT INTO mail_sent (id, from_addr, to_addrs, cc, bcc, reply_to, subject, html, body_text, '
-               'created_at, last_event, provider) VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT (id) DO NOTHING')
+               'created_at, last_event, provider, attachments) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT (id) DO NOTHING')
 INSERT_SENT_META = ('INSERT INTO mail_sent_meta (email_id, owner, is_auto, created_at) VALUES (?,?,0,?) '
                     'ON CONFLICT (email_id) DO NOTHING')
 
@@ -332,20 +333,23 @@ STORE = 'blob'
 
 
 def stored_attachments(url, token, ids):
-    """id -> (owner, attachments) for inbox rows already stored."""
+    """id -> (owner, attachments, table) for rows already stored, in either table."""
     if not ids:
         return {}
     marks = ','.join('?' * len(ids))
     results = execute(url, token, [
         {'sql': f'SELECT id, owner, attachments FROM mail_inbox WHERE id IN ({marks})', 'args': [arg(i) for i in ids]},
+        {'sql': f'SELECT s.id, m.owner, s.attachments FROM mail_sent s LEFT JOIN mail_sent_meta m ON m.email_id = s.id '
+                f'WHERE s.id IN ({marks})', 'args': [arg(i) for i in ids]},
     ])
     found = {}
-    for row in results[0]['response']['result']['rows']:
-        try:
-            entries = json.loads(row[2]['value'] or '[]')
-        except Exception:
-            entries = []
-        found[row[0]['value']] = ((row[1]['value'] or '').lower(), entries)
+    for table, result in zip(('mail_inbox', 'mail_sent'), results[:2]):
+        for row in result['response']['result']['rows']:
+            try:
+                entries = json.loads(row[2]['value'] or '[]')
+            except Exception:
+                entries = []
+            found[row[0]['value']] = ((row[1]['value'] or '').lower(), entries, table)
     return found
 
 
@@ -356,9 +360,26 @@ def arg(value, kind='text'):
 
 
 def execute(url, token, statements):
+    if url.startswith('file:'):
+        return execute_sqlite(url[len('file:'):], statements)
     return with_retry(lambda: execute_once(url, token, statements), 'turso')
 
 
+SQLITE_LOCK = threading.Lock()
+
+
+def execute_sqlite(path, statements):
+    """The pipeline's shape over a local SQLite file, for a database that is not served over HTTP."""
+    with SQLITE_LOCK, sqlite3.connect(path, timeout=120) as connection:
+        connection.execute('PRAGMA busy_timeout = 120000')
+        results = []
+        for statement in statements:
+            cursor = connection.execute(statement['sql'], [a['value'] for a in statement['args']])
+            rows = cursor.fetchall() if cursor.description else []
+            results.append({'response': {'result': {'rows': [[{'value': cell} for cell in row] for row in rows]}}})
+        return results
+
+
 def execute_once(url, token, statements):
     requests = [{'type': 'execute', 'stmt': stmt} for stmt in statements] + [{'type': 'close'}]
     request = urllib.request.Request(
@@ -409,6 +430,7 @@ def sent_statements(row):
             arg(row['id']), arg(row['from']), arg(json.dumps(row['to'])), arg(json.dumps(row['cc'])),
             arg(json.dumps(row['bcc'])), arg(json.dumps(row['replyTo'])), arg(row['subject']),
             arg(row['html']), arg(row['text']), arg(row['receivedAt']), arg('imported'), arg('mbox'),
+            arg(json.dumps(row['attachments'])),
         ]},
         {'sql': INSERT_SENT_META, 'args': [arg(row['id']), arg(row['owner']), arg(row['receivedAt'])]},
     ]
@@ -424,6 +446,7 @@ def main():
     parser.add_argument('--batch', type=int, default=10)
     parser.add_argument('--mark-read', action='store_true')
     parser.add_argument('--skip-attachments', action='store_true')
+    parser.add_argument('--kinds', default='inbox,sent', help='which rows the attachment pass touches: inbox, sent, or both')
     parser.add_argument('--attach-only', action='store_true',
                         help='second pass: upload attachments for rows already imported and write the URLs back')
     parser.add_argument('--workers', type=int, default=8, help='parallel attachment uploads')
@@ -450,8 +473,9 @@ def main():
         print(f'upload cap: {args.max_mbps} Mbit/s', flush=True)
     url, db_token = os.environ.get('TURSO_DATABASE_URL'), os.environ.get('TURSO_AUTH_TOKEN')
     blob_token = None if args.skip_attachments else (os.environ.get('BLOB_READ_WRITE_TOKEN') if STORE == 'blob' else 's3')
-    if not args.dry_run and not (url and db_token):
+    if not args.dry_run and not (url and (db_token or url.startswith('file:'))):
         sys.exit('TURSO_DATABASE_URL and TURSO_AUTH_TOKEN must be set')
+    kinds = {kind.strip() for kind in args.kinds.split(',') if kind.strip()}
     if not args.dry_run and not args.skip_attachments and not blob_token:
         print('BLOB_READ_WRITE_TOKEN unset: attachments will be metadata only', flush=True)
 
@@ -477,6 +501,7 @@ def main():
                 local['missing'] += 1
                 continue
             row['id'] = mine
+            row['table'] = stored[mine][2]
             entries = stored[mine][1]
             want = len(row['attachments'])
             complete = lambda e: bool(e.get('key')) or (bool(e.get('url')) and not args.replace_blob)
@@ -524,7 +549,7 @@ def main():
             touched.add(row['id'])
         for row in rows:
             if row['id'] in touched or row.get('reuse'):
-                execute(url, db_token, [{'sql': 'UPDATE mail_inbox SET attachments = ? WHERE id = ?',
+                execute(url, db_token, [{'sql': f'UPDATE {row["table"]} SET attachments = ? WHERE id = ?',
                                          'args': [arg(json.dumps(row['stored'])), arg(row['id'])]}])
                 local['updated'] += 1
             for attachment in row['attachments']:
@@ -659,7 +684,7 @@ def main():
             print(f'  {row["receivedAt"][:10]} {row["kind"]:5} {flag} {row["from"][:30]:30} {row["subject"][:44]}'
                   f'  [{len(row["attachments"])} files]', flush=True)
         if args.attach_only:
-            if row['kind'] != 'inbox' or not row['attachments']:
+            if row['kind'] not in kinds or not row['attachments']:
                 continue
             chunk.append(row)
             if len(chunk) >= args.batch:

package/app/api/mail/upload/route.ts

@@ -1,43 +0,0 @@
-import { NextResponse } from 'next/server'
-import { handleUpload, type HandleUploadBody } from '@vercel/blob/client'
-import { authenticate, isLocalOrigin, verifyMailAuth } from '@/lib/dev-auth'
-
-export const runtime = 'nodejs'
-
-const MAX_UPLOAD_BYTES = 100 * 1024 * 1024
-
-export async function POST(req: Request): Promise<NextResponse> {
-  const body = (await req.json()) as HandleUploadBody
-
-  try {
-    const result = await handleUpload({
-      body,
-      request: req,
-      onBeforeGenerateToken: async (_pathname, clientPayload) => {
-        // The session cookie is the normal path; the credentials in clientPayload are only
-        // still read so a tab that signed in before sessions existed can finish its upload.
-        let authorized = isLocalOrigin(req) || Boolean(await authenticate(req))
-        if (!authorized && clientPayload) {
-          try {
-            const creds = JSON.parse(clientPayload) as { email?: string; password?: string }
-            authorized = (await verifyMailAuth(creds.email ?? '', creds.password ?? '')).ok
-          } catch {
-            authorized = false
-          }
-        }
-        if (!authorized) throw new Error('Unauthorized')
-        return {
-          addRandomSuffix: true,
-          maximumSizeInBytes: MAX_UPLOAD_BYTES,
-          tokenPayload: null,
-        }
-      },
-      onUploadCompleted: async () => {
-        // No-op: the URL is returned to the client, which attaches it on send.
-      },
-    })
-    return NextResponse.json(result)
-  } catch (error) {
-    return NextResponse.json({ error: (error as Error).message }, { status: 400 })
-  }
-}