@freecorps/ember-email 0.5.7 → 0.5.8

package/README.md

@@ -71,6 +71,10 @@ export const signup = mutation({
   (with their size). Mind the provider's message size: for a large file, put a download link in the text
   (`ctx.storage.getUrl(file, { download, expiresIn })`) and pass it with `link: true`. The email then has no MIME part
   for it, and the message still holds the file (the thread lists it, and it is not collected).
+- **Retention:** `upsertMailbox({ retention: { closedDays: 730, spamDays: 30 } })` keeps closed and spam conversations
+  that many days, counted from the latest status change or message; a daily job (`lib:purge`, 04:00
+  UTC) then deletes the conversation and its messages, and file GC deletes their `.eml` and attachments. Without
+  `retention` (or with `retention: null`) nothing is deleted.
 
 ## Domains: `@freecorps/ember-email/cloudflare`
 

package/dist/client.d.ts

@@ -2,6 +2,11 @@ import type { FunctionReference } from "@freecorps/ember-server";
 import type { SendInput, SendResult } from "./component.ts";
 import type { EmailStatus } from "./events.ts";
 type Ref<T extends "query" | "mutation" | "action", A, R> = FunctionReference<T, "public", A, R>;
+/** Days closed and spam conversations are kept (`upsertMailbox({ retention })`); a missing kind is kept forever. */
+export type MailboxRetention = {
+    closedDays?: number;
+    spamDays?: number;
+};
 type Page<T> = {
     page: T[];
     isDone: boolean;
@@ -134,6 +139,7 @@ export interface EmailComponentApi {
                 html?: string;
             } | null;
             catchAll?: boolean | null;
+            retention?: MailboxRetention | null;
         }, string>;
         removeMailbox: Ref<"mutation", {
             name: string;
@@ -145,6 +151,7 @@ export interface EmailComponentApi {
             displayName?: string;
             autoReply?: unknown;
             catchAll?: boolean;
+            retention?: MailboxRetention;
         }>>;
         route: Ref<"query", {
             to: string;

package/dist/component.d.ts

@@ -513,6 +513,15 @@ export declare const emailFunctions: {
         } | null>;
         /** Cron: a send claimed more than LEASE_MS ago never finished (Worker died mid-send): fail it, never resend blind. */
         sweep: import("@freecorps/ember-server").RegisteredMutation<"internal", {}, number>;
+        /**
+         * Daily: deletes the closed and spam conversations past their mailbox's retention (with their messages; the
+         * messages' files are then collected by file GC). Bounded per run; schedules itself again while there is more.
+         */
+        purge: import("@freecorps/ember-server").RegisteredMutation<"internal", {}, {
+            threads: number;
+            messages: number;
+            more: boolean;
+        }>;
         applyEvents: import("@freecorps/ember-server").RegisteredMutation<"internal", {
             events: any[];
         }, ({
@@ -597,6 +606,10 @@ export declare const emailFunctions: {
                 html?: string;
             } | null;
             catchAll?: boolean | null;
+            retention?: {
+                closedDays?: number;
+                spamDays?: number;
+            } | null;
         }, GenericId<"mailboxes">>;
         removeMailbox: import("@freecorps/ember-server").RegisteredMutation<"public", {
             name: string;
@@ -615,6 +628,10 @@ export declare const emailFunctions: {
                 html?: string;
             };
             catchAll?: boolean;
+            retention?: {
+                closedDays?: number;
+                spamDays?: number;
+            };
         }[]>;
         route: import("@freecorps/ember-server").RegisteredQuery<"public", {
             to: string;
@@ -644,6 +661,7 @@ export declare const emailFunctions: {
             lastMessageAt: number;
             snippet: string;
             contactName?: string;
+            activityAt?: number;
             lastInboundAt?: number;
             assignee?: string;
             assigneeName?: string;
@@ -666,6 +684,7 @@ export declare const emailFunctions: {
                 lastMessageAt: number;
                 snippet: string;
                 contactName?: string;
+                activityAt?: number;
                 lastInboundAt?: number;
                 assignee?: string;
                 assigneeName?: string;
@@ -830,6 +849,7 @@ export declare const emailFunctions: {
                 lastMessageAt: number;
                 snippet: string;
                 contactName?: string;
+                activityAt?: number;
                 lastInboundAt?: number;
                 assignee?: string;
                 assigneeName?: string;
@@ -1322,6 +1342,15 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
         } | null>;
         /** Cron: a send claimed more than LEASE_MS ago never finished (Worker died mid-send): fail it, never resend blind. */
         sweep: import("@freecorps/ember-server").RegisteredMutation<"internal", {}, number>;
+        /**
+         * Daily: deletes the closed and spam conversations past their mailbox's retention (with their messages; the
+         * messages' files are then collected by file GC). Bounded per run; schedules itself again while there is more.
+         */
+        purge: import("@freecorps/ember-server").RegisteredMutation<"internal", {}, {
+            threads: number;
+            messages: number;
+            more: boolean;
+        }>;
         applyEvents: import("@freecorps/ember-server").RegisteredMutation<"internal", {
             events: any[];
         }, ({
@@ -1406,6 +1435,10 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
                 html?: string;
             } | null;
             catchAll?: boolean | null;
+            retention?: {
+                closedDays?: number;
+                spamDays?: number;
+            } | null;
         }, GenericId<"mailboxes">>;
         removeMailbox: import("@freecorps/ember-server").RegisteredMutation<"public", {
             name: string;
@@ -1424,6 +1457,10 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
                 html?: string;
             };
             catchAll?: boolean;
+            retention?: {
+                closedDays?: number;
+                spamDays?: number;
+            };
         }[]>;
         route: import("@freecorps/ember-server").RegisteredQuery<"public", {
             to: string;
@@ -1453,6 +1490,7 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
             lastMessageAt: number;
             snippet: string;
             contactName?: string;
+            activityAt?: number;
             lastInboundAt?: number;
             assignee?: string;
             assigneeName?: string;
@@ -1475,6 +1513,7 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
                 lastMessageAt: number;
                 snippet: string;
                 contactName?: string;
+                activityAt?: number;
                 lastInboundAt?: number;
                 assignee?: string;
                 assigneeName?: string;
@@ -1639,6 +1678,7 @@ export declare const email: import("@freecorps/ember-server").ComponentDefinitio
                 lastMessageAt: number;
                 snippet: string;
                 contactName?: string;
+                activityAt?: number;
                 lastInboundAt?: number;
                 assignee?: string;
                 assigneeName?: string;

package/dist/component.js

@@ -477,6 +477,7 @@ function ingest(ctx, a) {
             contact,
             ...((a.replyTo ?? a.from).name ? { contactName: (a.replyTo ?? a.from).name } : {}),
             status: spam ? "spam" : "open",
+            activityAt: ctx.now,
             token: token(ctx),
             messageCount: 0,
             unreadCount: 0,
@@ -513,6 +514,7 @@ function ingest(ctx, a) {
     const status = thread.status === "spam" ? "spam" : "open";
     ctx.db.patch(thread._id, {
         status,
+        activityAt: ctx.now,
         messageCount: thread.messageCount + 1,
         unreadCount: thread.unreadCount + 1,
         lastMessageAt: ctx.now,
@@ -601,10 +603,83 @@ function appendOutbound(ctx, thread, mb, a) {
     ctx.db.patch(thread._id, {
         messageCount: thread.messageCount + 1,
         lastMessageAt: ctx.now,
+        activityAt: ctx.now,
         ...(a.note ? {} : { status: thread.status === "spam" ? "spam" : "waiting", snippet }),
     });
     return id;
 }
+const DAY_MS = 86_400_000;
+const MAX_RETENTION_DAYS = 36_500;
+/** Rows (messages and threads) one purge transaction deletes before it schedules the rest. */
+const PURGE_BUDGET = 500;
+/** Candidates one purge run looks at per mailbox and status (some may not be due yet: see `purgeExpired`). */
+const PURGE_SCAN = 200;
+function retentionOf(r) {
+    const out = {};
+    for (const k of ["closedDays", "spamDays"]) {
+        const d = r[k];
+        if (d === undefined)
+            continue;
+        if (!Number.isInteger(d) || d < 1 || d > MAX_RETENTION_DAYS)
+            fail("ValidationError", `retention.${k}: a whole number of days in 1..${MAX_RETENTION_DAYS}`);
+        out[k] = d;
+    }
+    return out.closedDays === undefined && out.spamDays === undefined ? undefined : out;
+}
+/**
+ * One purge pass: the closed and spam threads whose latest status change or message (`activityAt`) is older than
+ * their mailbox's retention, straight from the `by_mailbox_status_activity` index (every candidate is due, so a
+ * thread kept longer never blocks the ones behind it). Threads from before `activityAt` (absent, first in the index)
+ * get it from `lastMessageAt` when they are not due yet, so they leave the head of the index.
+ */
+function purgeExpired(ctx) {
+    let budget = PURGE_BUDGET;
+    let threads = 0;
+    let messages = 0;
+    let more = false;
+    for (const mb of ctx.db.query("mailboxes").collect()) {
+        for (const [status, days] of [
+            ["closed", mb.retention?.closedDays],
+            ["spam", mb.retention?.spamDays],
+        ]) {
+            if (!days || more)
+                continue;
+            const cutoff = ctx.now - days * DAY_MS;
+            const due = ctx.db
+                .query("threads")
+                .withIndex("by_mailbox_status_activity", (q) => q.eq("mailbox", mb.name).eq("status", status).lte("activityAt", cutoff))
+                .take(PURGE_SCAN);
+            for (const t of due) {
+                if (budget <= 0) {
+                    more = true;
+                    break;
+                }
+                if (t.activityAt === undefined && t.lastMessageAt > cutoff) {
+                    ctx.db.patch(t._id, { activityAt: t.lastMessageAt });
+                    budget--;
+                    continue;
+                }
+                const batch = ctx.db.query("messages").withIndex("by_thread", (q) => q.eq("threadId", t._id)).take(budget);
+                for (const m of batch)
+                    ctx.db.delete(m._id);
+                messages += batch.length;
+                budget -= batch.length;
+                if (budget <= 0) {
+                    more = true; // the thread goes on the next run, once its last messages are gone
+                    break;
+                }
+                ctx.db.delete(t._id);
+                threads++;
+                budget--;
+            }
+            if (due.length === PURGE_SCAN)
+                more = true;
+        }
+    }
+    if (more)
+        ctx.scheduler.runAfter(0, "lib:purge", {});
+    return { threads, messages, more };
+}
 // ------------------------------------------------------------------------------------------------ functions
 const pageV = v.optional(v.object({ numItems: v.number(), cursor: v.union(v.string(), v.null()) }));
 export const emailFunctions = {
@@ -765,6 +840,14 @@ export const emailFunctions = {
                 return n;
             },
         }),
+        /**
+         * Daily: deletes the closed and spam conversations past their mailbox's retention (with their messages; the
+         * messages' files are then collected by file GC). Bounded per run; schedules itself again while there is more.
+         */
+        purge: internalMutation({
+            args: {},
+            handler: (ctx) => purgeExpired(ctx),
+        }),
         // ---------------------------------------------------------------- events
         applyEvents: internalMutation({
             args: { events: v.array(v.any()) },
@@ -852,6 +935,11 @@ export const emailFunctions = {
                 autoReply: v.optional(v.union(v.null(), v.object({ subject: v.string(), text: v.string(), html: v.optional(v.string()) }))),
                 /** The domain's catch-all (one per domain). Absent: keeps the stored value; false or null: clears it. */
                 catchAll: v.optional(v.union(v.null(), v.boolean())),
+                /**
+                 * Days closed and spam conversations are kept (whole days, 1..36500; a missing kind is kept forever).
+                 * Absent: keeps the stored policy; null: clears it (kept forever).
+                 */
+                retention: v.optional(v.union(v.null(), v.object({ closedDays: v.optional(v.number()), spamDays: v.optional(v.number()) }))),
             },
             handler: (ctx, a) => {
                 if (!/^[a-z][a-z0-9_-]{0,39}$/.test(a.name))
@@ -867,6 +955,7 @@ export const emailFunctions = {
                 const displayName = a.displayName === null ? undefined : (a.displayName ?? prev?.displayName);
                 const autoReply = a.autoReply === null ? undefined : (a.autoReply ?? prev?.autoReply);
                 const catchAll = a.catchAll === undefined ? !!prev?.catchAll : !!a.catchAll;
+                const retention = a.retention === null ? undefined : a.retention === undefined ? prev?.retention : retentionOf(a.retention);
                 if (catchAll) {
                     const other = catchAllOf(ctx, domainOf(address));
                     if (other && other.name !== a.name)
@@ -878,6 +967,7 @@ export const emailFunctions = {
                     ...(displayName ? { displayName } : {}),
                     ...(autoReply ? { autoReply } : {}),
                     ...(catchAll ? { catchAll: true } : {}),
+                    ...(retention ? { retention } : {}),
                     nextNumber: prev?.nextNumber ?? 1,
                     createdAt: prev?.createdAt ?? ctx.now,
                 };
@@ -1037,7 +1127,8 @@ export const emailFunctions = {
                 const t = ctx.db.get(a.threadId);
                 if (!t)
                     return fail("NotFound", "thread not found");
-                ctx.db.patch(a.threadId, { status: a.status });
+                if (t.status !== a.status)
+                    ctx.db.patch(a.threadId, { status: a.status, activityAt: ctx.now });
                 return a.status;
             },
         }),
@@ -1163,6 +1254,7 @@ export const emailFunctions = {
                     messageCount: 0,
                     unreadCount: 0,
                     lastMessageAt: ctx.now,
+                    activityAt: ctx.now,
                     snippet: snippetOf(a.text),
                 });
                 const thread = ctx.db.get(threadId);
@@ -1289,7 +1381,7 @@ export const email = defineComponent({
     bindings: (config) => (config?.transport ?? "binding") === "binding"
         ? [{ type: "send_email", name: config?.binding ?? "EMAIL", ...(config?.allowedSenders?.length ? { allowed_sender_addresses: config.allowedSenders } : {}) }]
         : [],
-    crons: (config) => cronJobs({ shard: config?.shard ?? "global:main" }).interval("sweep", { minutes: 10 }, "lib:sweep"),
+    crons: (config) => cronJobs({ shard: config?.shard ?? "global:main" }).interval("sweep", { minutes: 10 }, "lib:sweep").daily("purge", { hour: 4 }, "lib:purge"),
     queues: {
         events: { shard: (config) => config?.shard ?? "global:main", consumer: { maxBatchSize: 100, maxBatchTimeoutSeconds: 2, maxRetries: 10 }, handler: onEvents },
     },

package/dist/schema.d.ts

@@ -171,6 +171,17 @@ export declare const emailSchema: import("@freecorps/ember-server").SchemaDefini
         }> | undefined>>;
         /** Receives mail for any address of its domain that no other mailbox has (one per domain). */
         catchAll: import("@freecorps/ember-values").Validator<boolean, "optional", never>;
+        /**
+         * How long closed and spam conversations are kept, in days, counted from the latest status change or message
+         * (`threads.activityAt`). Absent: kept forever. A daily job deletes the conversation, its messages and their files.
+         */
+        retention: import("@freecorps/ember-values").Validator<{
+            closedDays?: number;
+            spamDays?: number;
+        }, "optional", NonNullable<import("@freecorps/ember-values").ObjectFieldPaths<{
+            closedDays: import("@freecorps/ember-values").Validator<number, "optional", never>;
+            spamDays: import("@freecorps/ember-values").Validator<number, "optional", never>;
+        }> | undefined>>;
         nextNumber: import("@freecorps/ember-values").Validator<number, "required", never>;
         createdAt: import("@freecorps/ember-values").Validator<number, "required", never>;
     }>, {
@@ -187,6 +198,11 @@ export declare const emailSchema: import("@freecorps/ember-server").SchemaDefini
         contactName: import("@freecorps/ember-values").Validator<string, "optional", never>;
         /** open | waiting | closed | spam */
         status: import("@freecorps/ember-values").Validator<string, "required", never>;
+        /**
+         * The latest status change or message: retention counts from it. Absent on threads from before 0.5.8 (the first
+         * purge fills it in from `lastMessageAt`).
+         */
+        activityAt: import("@freecorps/ember-values").Validator<number, "optional", never>;
         /** Plus-address token: replies to `support+t_<token>@` join this thread from any sender. */
         token: import("@freecorps/ember-values").Validator<string, "required", never>;
         messageCount: import("@freecorps/ember-values").Validator<number, "required", never>;
@@ -200,6 +216,7 @@ export declare const emailSchema: import("@freecorps/ember-server").SchemaDefini
     }>, {
         by_mailbox_last: ["mailbox", "lastMessageAt", "_creationTime"];
         by_mailbox_status: ["mailbox", "status", "lastMessageAt", "_creationTime"];
+        by_mailbox_status_activity: ["mailbox", "status", "activityAt", "_creationTime"];
         by_token: ["token", "_creationTime"];
         by_contact_subject: ["mailbox", "contact", "subjectKey", "_creationTime"];
     }, {

package/dist/schema.js

@@ -95,6 +95,11 @@ export const emailSchema = defineSchema({
             autoReply: v.optional(v.object({ subject: v.string(), text: v.string(), html: v.optional(v.string()) })),
             /** Receives mail for any address of its domain that no other mailbox has (one per domain). */
             catchAll: v.optional(v.boolean()),
+            /**
+             * How long closed and spam conversations are kept, in days, counted from the latest status change or message
+             * (`threads.activityAt`). Absent: kept forever. A daily job deletes the conversation, its messages and their files.
+             */
+            retention: v.optional(v.object({ closedDays: v.optional(v.number()), spamDays: v.optional(v.number()) })),
             nextNumber: v.number(),
             createdAt: v.number(),
         })
@@ -110,6 +115,11 @@ export const emailSchema = defineSchema({
             contactName: v.optional(v.string()),
             /** open | waiting | closed | spam */
             status: v.string(),
+            /**
+             * The latest status change or message: retention counts from it. Absent on threads from before 0.5.8 (the first
+             * purge fills it in from `lastMessageAt`).
+             */
+            activityAt: v.optional(v.number()),
             /** Plus-address token: replies to `support+t_<token>@` join this thread from any sender. */
             token: v.string(),
             messageCount: v.number(),
@@ -123,6 +133,7 @@ export const emailSchema = defineSchema({
         })
             .index("by_mailbox_last", ["mailbox", "lastMessageAt"])
             .index("by_mailbox_status", ["mailbox", "status", "lastMessageAt"])
+            .index("by_mailbox_status_activity", ["mailbox", "status", "activityAt"])
             .index("by_token", ["token"])
             .index("by_contact_subject", ["mailbox", "contact", "subjectKey"])
             .aggregate("unread", { by: ["mailbox"], count: true, sum: ["unreadCount"] }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecorps/ember-email",
-  "version": "0.5.7",
+  "version": "0.5.8",
   "description": "Email component for Ember on Cloudflare: send (atomic from mutations, scheduled, idempotent) through Email Service, templates, suppressions and per-project rate limits, delivery events from the Queue, inbound mail (Email Routing) parsed into mailboxes and threads. `@freecorps/ember-email/cloudflare` onboards a domain (sending, routing rules, event subscription) and removes it.",
   "keywords": [
     "ember",
@@ -52,9 +52,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@freecorps/ember-rate-limit": "0.5.7",
-    "@freecorps/ember-server": "0.5.7",
-    "@freecorps/ember-values": "0.5.7",
+    "@freecorps/ember-rate-limit": "0.5.8",
+    "@freecorps/ember-server": "0.5.8",
+    "@freecorps/ember-values": "0.5.8",
     "postal-mime": "^4.1.4"
   }
 }