@wirecat/cli-core 0.487.0 → 0.489.0

package/dist/calendar/writes.js

@@ -6,7 +6,7 @@ import { guardedWrite } from "../sends/guarded.js";
 import { DEFAULT_PERMISSIONS, levelFor } from "../sends/permissions.js";
 import { newOperationId } from "../sends/send-id.js";
 import { calendarIn, entryItem, eventIdOf, eventOf, plusDays } from "./reads.js";
-import { kindBoundLines, repeatRule } from "./recurrence.js";
+import { kindBoundLines, repeatRule, ruleCount, ruleEnd, ruleEndingBefore, ruleLine, splitDates } from "./recurrence.js";
 import { CALENDAR_WRITE_DEFAULTS, EVENTS_DELETE_SERIES_KEY, EVENTS_NOTIFY_KEY, } from "./writer.js";
 export const EVENT_RESPONSES = ["accepted", "tentative", "declined"];
 /** Each write is checked by its command's path, the key a declared CLI's `commandPermissions` already knows. */
@@ -15,6 +15,7 @@ const COMMAND_KEY = {
     update: "events.update",
     delete: "events.delete",
     respond: "events.respond",
+    move: "events.move",
 };
 const DATE = /^\d{4}-\d{2}-\d{2}$/u;
 const EMAIL = /^[^\s@]+@[^\s@]+\.[^\s@]+$/u;
@@ -150,27 +151,44 @@ const answerOf = (operationId, saved, repeats = false) => ({
 /** An id every provider takes as a client-chosen one: Google's base32hex `[a-v0-9]`, a CalDAV UID, a Graph id. */
 export const newEventId = () => randomUUID().replaceAll("-", "");
 const ID = /^[a-v0-9]{5,1024}$/u;
-export const createEvent = async (context, calendarReference, fields, options = {}) => {
-    const calendar = await calendarIn(context.store, context.accountId, calendarReference);
-    const draft = draftOf(fields, context.zone, { whole: true });
-    if (options.id !== undefined && !ID.test(options.id))
-        throw new CliError("validation_error", `${context.names.id} is the id a create's outcome_unknown named: 5 to 1024 of a-v and 0-9`, {
-            param: context.names.id,
-        });
-    const notify = notifyOf(context, options.notify, (draft.attendees?.length ?? 0) > 0);
-    const id = options.id ?? newEventId();
-    const guard = writeGuard(context, keysFor(context, "create", notify));
-    let entry;
+const idOf = (context, given) => {
+    if (given !== undefined && !ID.test(given))
+        throw new CliError("validation_error", `${context.names.id} is the id a create's outcome_unknown named: 5 to 1024 of a-v and 0-9`, { param: context.names.id });
+    return given ?? newEventId();
+};
+/** A create whose answer was lost names its id, so the owner's retry finds the event instead of making another. */
+const retryable = async (context, id, write) => {
     try {
-        entry = await guardedWrite(guard, { chatId: null, kind: "edit", key: COMMAND_KEY.create, operationId: id, sendId: id, notify }, () => context.writes.writer.create(calendar, draft, { id, notify }));
+        return await write();
     }
     catch (error) {
         if (isCliError(error) && error.code === "outcome_unknown")
-            throw new CliError("outcome_unknown", `${error.message} — repeat the create with ${context.names.id} ${id}: the provider then finds the event ` +
+            throw new CliError("outcome_unknown", `${error.message} — repeat with ${context.names.id} ${id}: the provider then finds the event ` +
                 "instead of making a second one", { ...error.details, operation_id: id });
         throw error;
     }
-    const [saved] = await context.store.saveEntries(calendar.id, [entry], context.now);
+};
+/**
+ * The store's save of what the provider already made. A failure here names the id: without it the owner's retry makes
+ * the event a second time.
+ */
+const savedAfterCreate = async (context, id, save) => {
+    try {
+        return await save();
+    }
+    catch (error) {
+        throw new CliError(isCliError(error) ? error.code : "invalid_response", `the provider made event ${id}, but the local store did not save it (${error instanceof Error ? error.message : String(error)}); ` +
+            `sync to store it — do not create it again: a create repeated with ${context.names.id} ${id} finds it`, { ...(isCliError(error) ? error.details : {}), operation_id: id, id });
+    }
+};
+export const createEvent = async (context, calendarReference, fields, options = {}) => {
+    const calendar = await calendarIn(context.store, context.accountId, calendarReference);
+    const draft = draftOf(fields, context.zone, { whole: true });
+    const id = idOf(context, options.id);
+    const notify = notifyOf(context, options.notify, (draft.attendees?.length ?? 0) > 0);
+    const guard = writeGuard(context, keysFor(context, "create", notify));
+    const entry = await retryable(context, id, () => guardedWrite(guard, { chatId: null, kind: "edit", key: COMMAND_KEY.create, operationId: id, sendId: id, notify }, () => context.writes.writer.create(calendar, draft, { id, notify })));
+    const [saved] = await savedAfterCreate(context, id, () => context.store.saveEntries(calendar.id, [entry], context.now));
     return answerOf(id, saved, draft.recurrence !== undefined);
 };
 const targetOf = async (context, reference, { calendar, scope }) => {
@@ -185,11 +203,10 @@ const targetOf = async (context, reference, { calendar, scope }) => {
         throw new CliError("validation_error", `${reference} is on calendars ${live.map(({ calendarId }) => calendarId).join(", ")}; name one with ` +
             `${context.names.calendar}`, { param: context.names.calendar });
     let target = entry;
-    if (scope === "series") {
+    if (scope !== "this") {
+        const flag = scope === "series" ? context.names.series : context.names.following;
         if (entry.kind !== "occurrence" || !entry.seriesExternalId)
-            throw new CliError("validation_error", `${reference} does not repeat; leave out ${context.names.series}`, {
-                param: context.names.series,
-            });
+            throw new CliError("validation_error", `${reference} does not repeat; leave out ${flag}`, { param: flag });
         const series = await context.store.entry(entry.calendarId, entry.seriesExternalId);
         if (!series)
             throw new CliError("not_found", `the series of ${reference} is not stored; sync and try again`);
@@ -207,7 +224,39 @@ const targetOf = async (context, reference, { calendar, scope }) => {
         participants: details.participants.filter(({ entryId }) => entryId === entry.id),
     };
 };
+const scopeOf = (context, { series, following }) => {
+    if (series && following)
+        throw new CliError("validation_error", `choose ${context.names.series} or ${context.names.following}, not both`, {
+            param: context.names.following,
+        });
+    return following ? "following" : series ? "series" : "this";
+};
+/**
+ * A new COUNT or UNTIL that drops slots the series had deletes them as surely as a delete of the series does, so it
+ * takes that key too. A bound this cannot compare with the old one counts as shorter.
+ */
+const shortens = (target, rule, zone) => {
+    // A date UNTIL takes in its whole day.
+    const endOf = (line, in_) => {
+        const end = ruleEnd(line, in_);
+        return end !== undefined && /UNTIL=\d{8}(?:;|$)/iu.test(line) ? end + 86_400_000 - 1 : end;
+    };
+    const [until, count] = [endOf(rule, zone), ruleCount(rule)];
+    if (until === undefined && count === undefined)
+        return false;
+    const before = ruleLine(target.recurrence ?? []);
+    const [wasUntil, wasCount] = before === undefined ? [] : [endOf(before, target.timezone ?? zone), ruleCount(before)];
+    if (until !== undefined)
+        return wasUntil === undefined || until < wasUntil;
+    return wasCount === undefined || Number(count) < Number(wasCount);
+};
 export const updateEvent = async (context, reference, fields, options = {}) => {
+    if (scopeOf(context, options) === "following")
+        return updateFollowing(context, reference, fields, options);
+    if (options.id !== undefined)
+        throw new CliError("validation_error", `${context.names.id} goes with ${context.names.following} only`, {
+            param: context.names.id,
+        });
     const scope = options.series ? "series" : "this";
     if (fields.repeat !== undefined && scope !== "series")
         throw new CliError("validation_error", `a repeat rule belongs to the whole series: add ${context.names.series}`, {
@@ -226,12 +275,15 @@ export const updateEvent = async (context, reference, fields, options = {}) => {
                 ? "change its kind in the calendar itself"
                 : `give its rule again with ${context.names.repeat} and UNTIL as a ${target.allDay ? "UTC time" : "date"}`), { reason: "event.repeat-kind" });
     const notify = notifyOf(context, options.notify, hasOthers(participants) || (patch.attendees?.length ?? 0) > 0);
+    const ends = patch.recurrence !== undefined && shortens(target, patch.recurrence[0], context.zone);
     const operationId = newOperationId();
-    const entry = await guardedWrite(writeGuard(context, keysFor(context, "update", notify)), { chatId: null, kind: "edit", key: COMMAND_KEY.update, operationId, notify }, () => context.writes.writer.update(calendar, target.externalId, patch, { scope, etag: target.etag, notify }));
+    const entry = await guardedWrite(writeGuard(context, [...keysFor(context, "update", notify), ...(ends ? [EVENTS_DELETE_SERIES_KEY] : [])]), { chatId: null, kind: "edit", key: COMMAND_KEY.update, operationId, notify }, () => context.writes.writer.update(calendar, target.externalId, patch, { scope, etag: target.etag, notify }));
     const [saved] = await context.store.saveEntries(calendar.id, [entry], context.now);
     return answerOf(operationId, saved, patch.recurrence !== undefined || (switching && scope === "series"));
 };
 export const deleteEvent = async (context, reference, options = {}) => {
+    if (scopeOf(context, options) === "following")
+        return deleteFollowing(context, reference, options);
     const scope = options.series ? "series" : "this";
     const { calendar, entry, target, participants } = await targetOf(context, reference, { ...options, scope });
     const notify = notifyOf(context, options.notify, hasOthers(participants));
@@ -261,3 +313,190 @@ export const respondToEvent = async (context, reference, response, options = {})
     const [saved] = await context.store.saveEntries(calendar.id, [entry], context.now);
     return answerOf(operationId, saved);
 };
+const midnight = (date) => Date.parse(`${date}T00:00:00Z`);
+const splitOf = (context, reference, { entry, target }) => {
+    if (entry.originalStartsAt === null)
+        throw new CliError("not_found", `the slot of ${reference} in its series is not stored; sync and try again`);
+    const at = entry.originalStartsAt;
+    const rule = ruleLine(target.recurrence ?? []);
+    if (!rule)
+        throw new CliError("validation_error", `the series of ${reference} repeats by single dates only; change it in the calendar itself`, { reason: "event.repeat-split" });
+    const zone = target.timezone ?? context.zone;
+    const end = ruleEnd(rule, zone);
+    const first = target.allDay
+        ? target.startDate !== null && at <= midnight(target.startDate)
+        : at <= (target.startsAt ?? 0);
+    return { at, first, rule, ended: end !== undefined && end < at, zone };
+};
+/** The old series' rule ending before the split, with the dates and other lines that belong before it. */
+const endedRecurrence = (target, split) => [
+    ruleEndingBefore(split.rule, split.at, { allDay: target.allDay }),
+    ...(target.recurrence ?? []).filter((line) => !/^(RRULE|EXDATE|RDATE)[:;]/iu.test(line)),
+    ...splitDates(target.recurrence ?? [], split.at, split.zone).before,
+];
+/** Each later write of a split carries what was done already, so the owner's retry completes it without a double. */
+const afterCreate = async (context, id, write) => {
+    try {
+        return await write();
+    }
+    catch (error) {
+        if (!isCliError(error))
+            throw error;
+        // A lost answer may have ended the old series: after a sync this occurrence is gone, and a retry finds nothing.
+        const next = error.code === "outcome_unknown"
+            ? `sync: if the series now ends before this occurrence, the split is done; if not, repeat with ` +
+                `${context.names.id} ${id}, which finds the new series instead of making it again`
+            : `repeat with ${context.names.id} ${id}: the new series is found, not made again, and the old one ends`;
+        throw new CliError(error.code, `${error.message}; the new series ${id} is made — ${next}`, {
+            ...error.details,
+            operation_id: id,
+        });
+    }
+};
+/**
+ * "This and all following" from the writer's own writes, so every provider has it: neither RFC 5545 nor Google's API has
+ * such a change (https://developers.google.com/workspace/calendar/api/guides/recurringevents only warns against
+ * changing the instances one by one for it). The new series is made first, under an id journaled before the call, and then the old one ends before the slot:
+ * a conflict on the old series stops nothing being lost, and a retry with the id finds the new series. Until then
+ * later slots show twice, never none.
+ */
+const updateFollowing = async (context, reference, fields, options) => {
+    const found = await targetOf(context, reference, { ...options, scope: "following" });
+    const { calendar, entry, target, participants } = found;
+    const split = splitOf(context, reference, found);
+    if (split.first && !split.ended) {
+        const { following: _following, id: _id, ...rest } = options;
+        return updateEvent(context, reference, fields, { ...rest, series: true });
+    }
+    const patch = draftOf(fields, context.zone, { whole: false, allDay: target.allDay });
+    if ((patch.startDate !== undefined && !target.allDay) || (patch.startsAt !== undefined && target.allDay))
+        throw new CliError("validation_error", `a switch between timed and all-day changes the whole series: use ${context.names.series}`, { param: context.names.series, reason: "event.repeat-kind" });
+    const count = ruleCount(split.rule);
+    if (count !== undefined && !patch.recurrence)
+        throw new CliError("validation_error", `the series repeats COUNT=${count} times, and only the provider knows how many come from this occurrence on; ` +
+            `give the new part its own rule with ${context.names.repeat}, ending with UNTIL or its own COUNT`, { param: context.names.repeat, reason: "event.repeat-count" });
+    if (split.ended && options.id === undefined)
+        throw new CliError("validation_error", `the series of ${reference} already ends before it; if a split ended in outcome_unknown, repeat it with ` +
+            `${context.names.id} and the id it named`, { param: context.names.id, reason: "event.repeat-split" });
+    const id = idOf(context, options.id);
+    const draft = {
+        title: patch.title ?? target.title ?? "",
+        ...(patch.startsAt !== undefined || patch.startDate !== undefined
+            ? { startsAt: patch.startsAt, endsAt: patch.endsAt, startDate: patch.startDate, endDate: patch.endDate }
+            : target.allDay
+                ? {
+                    startDate: new Date(split.at).toISOString().slice(0, 10),
+                    endDate: new Date(split.at + midnight(target.endDate) - midnight(target.startDate))
+                        .toISOString()
+                        .slice(0, 10),
+                }
+                : { startsAt: split.at, endsAt: split.at + (target.endsAt ?? 0) - (target.startsAt ?? 0) }),
+        // A repeating event needs its zone at Google; the caller's stands in for a series stored without one.
+        timezone: patch.timezone ?? target.timezone ?? context.zone,
+        location: patch.location === undefined ? target.location : patch.location,
+        description: patch.description === undefined ? target.description : patch.description,
+        attendees: patch.attendees ??
+            participants
+                .filter(({ self }) => !self)
+                .map(({ email, optional }) => ({ email, ...(optional ? { optional: true } : {}) })),
+        recurrence: [
+            patch.recurrence?.[0] ?? split.rule,
+            ...splitDates(target.recurrence ?? [], split.at, split.zone).after,
+        ],
+    };
+    for (const key of ["startsAt", "endsAt", "startDate", "endDate"])
+        if (draft[key] === undefined)
+            delete draft[key];
+    const notify = notifyOf(context, options.notify, hasOthers(participants) || (patch.attendees?.length ?? 0) > 0);
+    const guard = writeGuard(context, [
+        ...new Set([...keysFor(context, "create", notify), ...keysFor(context, "update", notify)]),
+    ]);
+    const made = await retryable(context, id, () => guardedWrite(guard, { chatId: null, kind: "edit", key: COMMAND_KEY.update, operationId: id, sendId: id, notify }, () => context.writes.writer.create(calendar, draft, { id, notify })));
+    const written = [made];
+    if (!split.ended)
+        written.push(await afterCreate(context, id, () => guardedWrite(guard, { chatId: null, kind: "edit", key: COMMAND_KEY.update, operationId: newOperationId(), notify }, () => context.writes.writer.update(calendar, target.externalId, { recurrence: endedRecurrence(target, split) }, { scope: "series", etag: target.etag, notify }))));
+    const [saved, ended] = await savedAfterCreate(context, id, () => context.store.saveEntries(calendar.id, written, context.now));
+    return {
+        operation_id: id,
+        event: entry.eventId === null ? null : `event:${entry.eventId}`,
+        scope: "following",
+        entry: saved ? entryItem(saved) : null,
+        ended: ended ? entryItem(ended) : null,
+        hint: "sync to store the occurrences of both series",
+    };
+};
+/**
+ * The series ends before the occurrence. The earlier ones stay, so it takes the key of a deletion that asks, not the
+ * series'.
+ */
+const deleteFollowing = async (context, reference, options) => {
+    const found = await targetOf(context, reference, { ...options, scope: "following" });
+    const { calendar, entry, target, participants } = found;
+    const split = splitOf(context, reference, found);
+    if (split.first) {
+        const { following: _following, ...rest } = options;
+        return deleteEvent(context, reference, { ...rest, series: true });
+    }
+    if (split.ended)
+        throw new CliError("validation_error", `the series of ${reference} already ends before it; sync to see it`, {
+            reason: "event.repeat-split",
+        });
+    const notify = notifyOf(context, options.notify, hasOthers(participants));
+    const operationId = newOperationId();
+    const recurrence = endedRecurrence(target, split);
+    const written = await guardedWrite(writeGuard(context, keysFor(context, "update", notify)), { chatId: null, kind: "edit", key: COMMAND_KEY.delete, operationId, notify }, () => context.writes.writer.update(calendar, target.externalId, { recurrence }, { scope: "series", etag: target.etag, notify }));
+    const [ended] = await context.store.saveEntries(calendar.id, [written], context.now);
+    const removed = await context.store.removeEntries(calendar.id, [entry.externalId], context.now);
+    return {
+        operation_id: operationId,
+        event: entry.eventId === null ? null : `event:${entry.eventId}`,
+        scope: "following",
+        removed,
+        ended: ended ? entryItem(ended) : null,
+        ...(ruleCount(split.rule) === undefined ? {} : { converted: `COUNT became ${recurrence[0]?.slice(6)}` }),
+        hint: "sync to cancel the later occurrences stored",
+    };
+};
+const WRITABLE = new Set(["owner", "writer"]);
+/**
+ * The event, under the same id, to another calendar of the same account. A repeating event moves whole: an
+ * occurrence alone cannot change calendar at Google, and moving one copy of a series apart would split it.
+ */
+export const moveEvent = async (context, reference, toCalendar, options = {}) => {
+    const move = context.writes.writer.move?.bind(context.writes.writer);
+    if (!move)
+        throw new CliError("configuration_error", `${context.writes.writer.provider} cannot move events`);
+    const { calendar, target, participants } = await targetOf(context, reference, {
+        ...options,
+        scope: options.series ? "series" : "this",
+    });
+    if (target.kind === "occurrence")
+        throw new CliError("validation_error", `${reference} is one occurrence of a repeating event, which moves whole: add ${context.names.series}`, { param: context.names.series });
+    const destination = await calendarIn(context.store, context.accountId, toCalendar);
+    if (destination.id === calendar.id)
+        throw new CliError("validation_error", `${reference} is on calendar ${calendar.id} already`);
+    if (!WRITABLE.has(destination.accessRole ?? ""))
+        throw new CliError("permission_error", `calendar ${destination.id} takes no new events from this account`, {
+            reason: "calendar.read-only",
+        });
+    if (target.eventType && target.eventType !== "default")
+        // https://developers.google.com/workspace/calendar/api/v3/reference/events/move: only default events move.
+        throw new CliError("validation_error", `a ${target.eventType} event stays on its calendar`, {
+            reason: "event.unmovable",
+        });
+    const notify = notifyOf(context, options.notify, hasOthers(participants));
+    const operationId = newOperationId();
+    const moved = await guardedWrite(writeGuard(context, keysFor(context, "move", notify)), { chatId: null, kind: "edit", key: COMMAND_KEY.move, operationId, notify }, () => move(calendar, target.externalId, destination, { etag: target.etag, notify }));
+    await context.store.removeEntries(calendar.id, [target.externalId], context.now);
+    // The shared UID joins the new copy to the same event as the cancelled one; a provider may leave it out of a move.
+    const [saved] = await context.store.saveEntries(destination.id, [{ ...moved, icalUid: moved.icalUid ?? target.icalUid }], context.now);
+    return {
+        ...answerOf(operationId, saved, target.kind === "series"),
+        from_calendar: String(calendar.id),
+        calendar: String(destination.id),
+        // Stored once here, the copy would stay as it is now: no sync reads a calendar that does not sync.
+        ...(destination.synced
+            ? {}
+            : { hint: `calendar ${destination.id} does not sync: calendars sync ${destination.id} keeps the event current` }),
+    };
+};

package/dist/errors/reasons.d.ts

@@ -7,7 +7,7 @@ export declare const REASONS: ErrorReason[];
  * Detail keys an error from this package may carry, until each is registered on its own reason. Not `cause`:
  * that can hold a provider's or SQLite's raw text.
  */
-export declare const DETAILS: readonly ["contracts", "allowed", "alternative", "arguments", "at", "available", "batch", "budget", "built", "candidates", "chat", "chat_id", "cli", "complete", "emitted_bytes", "exact", "forum", "guide", "hint", "id", "index", "kind", "limit", "limit_source", "max_bytes", "model", "name", "operation_id", "operation_ids", "text_limit", "store_refused_in", "partial_output", "path", "permission", "person_ids", "previous_chat_id", "profiles", "provider_error", "provider", "recovery", "reference", "remembered", "retry_at", "runtime", "saved", "scheduled_for", "script", "sent", "sent_count", "settlement_failed", "span", "sqlite", "stage", "standing", "state", "stderr", "term", "total", "upgraded", "wanted"];
+export declare const DETAILS: readonly ["contracts", "calendars", "allowed", "alternative", "arguments", "at", "available", "batch", "budget", "built", "candidates", "chat", "chat_id", "cli", "complete", "emitted_bytes", "exact", "forum", "guide", "hint", "id", "index", "kind", "limit", "limit_source", "max_bytes", "model", "name", "operation_id", "operation_ids", "text_limit", "store_refused_in", "partial_output", "path", "permission", "person_ids", "previous_chat_id", "profiles", "provider_error", "provider", "recovery", "reference", "remembered", "retry_at", "runtime", "saved", "scheduled_for", "script", "sent", "sent_count", "settlement_failed", "span", "sqlite", "stage", "standing", "state", "stderr", "term", "total", "upgraded", "wanted"];
 /** What a CLI adds to the shared registry: its own reasons, and the detail keys its adapters raise. */
 export interface AppErrors {
     reasons?: readonly ErrorReason[];

package/dist/errors/reasons.js

@@ -59,6 +59,10 @@ const OTHERS = [
     validation("event.notify-required", "the event has other attendees; say whether to notify them"),
     validation("event.repeat-invalid", "the repeat rule is outside the RFC 5545 subset the calendars take"),
     validation("event.repeat-kind", "the series' rule holds a date or a time of the kind the event is leaving"),
+    validation("event.repeat-split", "the series cannot be split at this occurrence, or was split there already"),
+    validation("event.repeat-count", "a COUNT series is split only with a rule for its new part"),
+    validation("event.unmovable", "the provider keeps this kind of event on its calendar"),
+    ["calendar.read-only", "permission_error", "the calendar takes no new events from this account"],
     ["busy.offline", "permission_error", "calendars busy asks the provider; --offline was given"],
     validation("busy.no-calendars", "no calendar syncs and no person was named, so there is nobody to ask"),
     validation("store.refused", "the local store refused the document text"),
@@ -150,6 +154,7 @@ export const REASONS = defineReasons([
  */
 export const DETAILS = [
     "contracts",
+    "calendars",
     "allowed",
     "alternative",
     "arguments",

package/dist/mail/attachments.d.ts

@@ -0,0 +1,15 @@
+import type { LoadEngine } from "../readers/types.js";
+import type { AttachmentInput, EmailAddress } from "../store/index.js";
+import type { Address, ParsedMail } from "./parse.js";
+import type { MailSyncResult } from "./sync.js";
+export declare const MAX_ATTACHMENT_CHARS = 200000;
+/** One chunk of raw mail is held in memory at a time. */
+export declare const FETCH_CHUNK = 50;
+export declare const addressOf: ({ email, name }: Address) => EmailAddress;
+export type AttachmentLog = NonNullable<MailSyncResult["attachments"]>;
+/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
+export declare const readAttachments: (mail: ParsedMail, messageId: string, { loadEngine, maxAttachmentChars, at }: {
+    loadEngine: LoadEngine;
+    maxAttachmentChars: number;
+    at: number;
+}, log: AttachmentLog) => Promise<AttachmentInput[]>;

package/dist/mail/attachments.js

@@ -0,0 +1,38 @@
+import { extractText } from "../attachments/extract.js";
+export const MAX_ATTACHMENT_CHARS = 200_000;
+/** One chunk of raw mail is held in memory at a time. */
+export const FETCH_CHUNK = 50;
+export const addressOf = ({ email, name }) => ({ address: email, name });
+/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
+export const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at }, log) => {
+    const files = [];
+    for (const [position, part] of mail.attachments.entries()) {
+        const extraction = part.bytes === null
+            ? { status: "unavailable" }
+            : await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine);
+        log.push({ message: messageId, part: part.id, status: extraction.status });
+        const extracted = extraction.status === "extracted" ? extraction : undefined;
+        files.push({
+            position,
+            kind: "file",
+            mime: part.mime,
+            name: part.name,
+            title: null,
+            url: null,
+            size: part.bytes === null ? null : part.bytes.byteLength,
+            width: null,
+            height: null,
+            duration: null,
+            providerRef: { message: messageId, part: part.id },
+            localPath: null,
+            text: extracted ? extracted.text.slice(0, maxAttachmentChars) : null,
+            normalizedText: null,
+            extraction: extracted ? "text" : "failed",
+            extractor: extracted ? extracted.extractor : null,
+            extractionError: extracted ? null : extraction.status,
+            contentSha256: null,
+            extractedAt: at,
+        });
+    }
+    return files;
+};

package/dist/mail/locations.d.ts

@@ -0,0 +1,28 @@
+import type { FolderEmail, MessageStore } from "../store/index.js";
+import type { Quick, Searched } from "./source.js";
+/** The flags a listing shows that differ from the stored ones, by Message-ID, for mail whose UID is stored. */
+export declare const flagsToRefresh: (rows: ReadonlyMap<string, FolderEmail>, listing: readonly Quick[]) => Map<string, {
+    read?: boolean;
+    flagged?: boolean;
+}>;
+export interface LocationProof {
+    /** Each folder listed by UID: what the server searched, and the stored mail with a UID there, before this run saved. */
+    folders: ReadonlyMap<string, {
+        searched: Searched;
+        rows: readonly FolderEmail[];
+    }>;
+    /** Mail received before this is outside the window IMAP's SINCE can vouch for. */
+    windowFrom: number;
+    /** Messages this run saw in any folder: one moved to another folder is not gone. */
+    present: ReadonlySet<string>;
+    now: number;
+}
+/**
+ * A folder listed by UID proves a deletion from the UIDs it stores: mail in the window whose UID the server no
+ * longer lists is gone from that folder, and gone altogether when no other location of it is left. Nothing is
+ * changed when too many would go, as for a header listing.
+ */
+export declare const proveByLocation: (store: MessageStore, accountId: number, { folders, windowFrom, present, now }: LocationProof, refuse: (gone: number, candidates: number) => string | undefined) => Promise<{
+    deleted: number;
+    skipped?: string;
+}>;

package/dist/mail/locations.js

@@ -0,0 +1,54 @@
+/** The flags a listing shows that differ from the stored ones, by Message-ID, for mail whose UID is stored. */
+export const flagsToRefresh = (rows, listing) => {
+    const changes = new Map();
+    for (const { uid, seen, flagged } of listing) {
+        const row = rows.get(uid);
+        if (row === undefined)
+            continue;
+        const changed = {
+            ...(seen === undefined || seen === row.read ? {} : { read: seen }),
+            ...(flagged === undefined || flagged === row.flagged ? {} : { flagged }),
+        };
+        if (Object.keys(changed).length)
+            changes.set(row.externalId, changed);
+    }
+    return changes;
+};
+/**
+ * A folder listed by UID proves a deletion from the UIDs it stores: mail in the window whose UID the server no
+ * longer lists is gone from that folder, and gone altogether when no other location of it is left. Nothing is
+ * changed when too many would go, as for a header listing.
+ */
+export const proveByLocation = async (store, accountId, { folders, windowFrom, present, now }, refuse) => {
+    const candidates = new Set();
+    const lost = new Map();
+    const lostUids = new Map();
+    for (const [folder, { searched, rows }] of folders) {
+        const inWindow = new Set(searched.uids.map(Number));
+        for (const row of rows) {
+            if (row.receivedAt === null || row.receivedAt < windowFrom)
+                continue;
+            candidates.add(row.externalId);
+            if (inWindow.has(row.uid))
+                continue;
+            const entry = lost.get(row.externalId) ?? { row, locations: 0, folders: new Set() };
+            entry.locations++;
+            entry.folders.add(folder);
+            lost.set(row.externalId, entry);
+            const uids = lostUids.get(folder) ?? [];
+            uids.push(row.uid);
+            lostUids.set(folder, uids);
+        }
+    }
+    const gone = [...lost]
+        .filter(([id, entry]) => !present.has(id) && entry.locations === entry.row.locations && entry.folders.size >= entry.row.folders)
+        .map(([id]) => id);
+    const refused = refuse(gone.length, candidates.size);
+    if (refused)
+        return { deleted: 0, skipped: refused };
+    // Marked first: an interruption leaves locations that prove it again, never an email with no way to be found.
+    const deleted = gone.length ? await store.mail.markDeleted(accountId, gone, now) : 0;
+    for (const [folder, uids] of lostUids)
+        await store.mail.forgetLocations(accountId, folder, folders.get(folder).searched.validity, uids);
+    return { deleted };
+};

package/dist/mail/quick.d.ts

@@ -20,6 +20,8 @@ export interface QuickPlan {
     asked: number;
 }
 export interface QuickRead {
+    /** Every message of the window the server listed, done or not: its flags refresh the stored mail. */
+    listing: Quick[];
     /** Messages read and identified, with their raw bytes so the save does not read them again. */
     read: {
         item: Listed;
@@ -36,4 +38,4 @@ export interface QuickRead {
  * One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
  * the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
  */
-export declare const readQuick: (source: MailSource & Required<Pick<MailSource, "quick" | "identify">>, since: Date, folder: string, searched: Searched, done: Ranges, max: number, plan: QuickPlan | undefined) => Promise<QuickRead>;
+export declare const readQuick: (source: MailSource & Required<Pick<MailSource, "quick" | "identify">>, since: Date, folder: string, searched: Searched, done: Ranges, max: number, plan: QuickPlan | undefined, located: ReadonlySet<string>) => Promise<QuickRead>;

package/dist/mail/quick.js

@@ -6,16 +6,18 @@ export const keyOf = (folder, uid) => `${folder}\0${uid}`;
  * One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
  * the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
  */
-export const readQuick = async (source, since, folder, searched, done, max, plan) => {
-    const answer = { read: [], unreadable: [], capped: false };
+export const readQuick = async (source, since, folder, searched, done, max, plan, located) => {
+    const answer = { listing: [], read: [], unreadable: [], capped: false };
     let items = plan?.quick.get(folder);
     if (items === undefined) {
-        const asked = searched.uids.filter((uid) => !within(done, Number(uid)) && (plan?.only === undefined || plan.only.has(keyOf(folder, uid))));
+        // Mail whose location is known is listed for its flags only; it is never read again.
+        const asked = searched.uids.filter((uid) => plan?.only === undefined || plan.only.has(keyOf(folder, uid)));
+        const wanted = asked.filter((uid) => !within(done, Number(uid)) && !located.has(uid));
         if (asked.length > 0 && plan) {
             const stop = plan.budget.take();
             if (stop) {
                 plan.stopped = stop;
-                plan.remaining += asked.length;
+                plan.remaining += wanted.length;
                 return { ...answer, capped: true };
             }
         }
@@ -23,15 +25,16 @@ export const readQuick = async (source, since, folder, searched, done, max, plan
         if (plan !== undefined) {
             plan.quick.set(folder, items);
             const got = new Set(items.map(({ uid }) => uid));
-            for (const uid of asked)
+            for (const uid of wanted)
                 if (!got.has(uid)) {
                     plan.failed.add(keyOf(folder, uid));
                     plan.unlisted.add(keyOf(folder, uid));
                 }
         }
     }
+    answer.listing = items;
     const open = items
-        .filter(({ uid }) => !within(done, Number(uid)) && !plan?.tried.has(keyOf(folder, uid)))
+        .filter(({ uid }) => !within(done, Number(uid)) && !located.has(uid) && !plan?.tried.has(keyOf(folder, uid)))
         .sort((a, b) => Number(b.uid) - Number(a.uid));
     const slice = open.slice(0, plan === undefined ? max : (source.fetchBatch ?? FETCH_SLICE));
     if (plan !== undefined)

package/dist/mail/resume.d.ts

@@ -30,6 +30,12 @@ export interface ResumePoint {
         total: number;
         at: string;
     };
+    /**
+     * From this date on, every stored email of the folder has its UID in `email_locations`: a listing that gives UIDs
+     * and flags is enough to refresh flags and prove deletions. Absent after an engine that did not know locations
+     * wrote the point, and then one header listing records them.
+     */
+    located?: string;
     /** What the latest run listed, whatever its window. */
     last?: {
         since: string;

package/dist/mail/retry.d.ts

@@ -0,0 +1,23 @@
+import type { LoadEngine } from "../readers/types.js";
+import type { MessageStore } from "../store/index.js";
+import type { MailSource } from "./source.js";
+import type { MailAccount, MailSyncOptions, MailSyncResult } from "./sync.js";
+/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
+export declare const RETRYABLE_ATTACHMENTS: string[];
+/**
+ * Reads again, by Gmail id, only the stored emails with an attachment a missing reader left unread: memo keeps no
+ * attachment file, so the email comes from the source again. Nothing is listed, so no deletion is proved and no
+ * resume point moves; the email keeps its stored mailboxes, flags and rule.
+ */
+export declare const retryUnread: ({ store, source, account, since, max, maxBodyChars, maxAttachmentChars, now, loadEngine, index, }: {
+    store: MessageStore;
+    source: MailSource;
+    account: MailAccount;
+    since: Date;
+    max: number;
+    maxBodyChars: number;
+    maxAttachmentChars: number;
+    now: () => number;
+    loadEngine: LoadEngine;
+    index: MailSyncOptions["index"];
+}) => Promise<MailSyncResult>;