@dxos/plugin-jmap 0.10.0

package/src/operations/mail/sync/sync-provider.ts

@@ -0,0 +1,481 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Chunk from 'effect/Chunk';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Predicate from 'effect/Predicate';
+import * as Stream from 'effect/Stream';
+
+import { type Resolver, resolve } from '@dxos/extractor';
+import { Cursor } from '@dxos/link';
+import { log } from '@dxos/log';
+import { Stage } from '@dxos/pipeline';
+import { EmailStage } from '@dxos/pipeline-email';
+import * as Mailbox from '@dxos/plugin-inbox/Mailbox';
+import { MailSyncError, type MailSyncItem, MailSyncProvider, type MailSyncSource } from '@dxos/plugin-inbox/sync';
+import type * as SyncStreamConfig from '@dxos/plugin-inbox/SyncStreamConfig';
+import * as SystemTags from '@dxos/plugin-inbox/SystemTags';
+import { TagIndex } from '@dxos/schema';
+import { Person } from '@dxos/types';
+
+import { Jmap, JmapMail } from '../../../apis';
+import { JMAP_DOMAIN } from '../../../constants';
+import { type JmapApiError } from '../../../errors';
+import { JmapMailApi } from '../../../services';
+import { type AttachmentMetadata, decodeBody, mapToMessage } from '../mapper';
+import { findOrCreateJmapTag } from '../tags';
+import { JMAP_KEYWORD_TAGS, JMAP_ROLE_TAGS } from './system-tags';
+
+/** The resolved delta for one run — either a fresh capture (no delta) or a fetched `Email/changes` chunk. */
+type DeltaPlan = {
+  readonly token: string | undefined;
+  readonly createdIds: readonly string[] | undefined;
+  readonly updatedIds: readonly string[];
+  readonly hasMoreDelta: boolean;
+};
+
+const MAIL_ACCOUNT_CAPABILITY = 'urn:ietf:params:jmap:mail';
+
+/** JMAP mail's streaming-pipeline tuning; see {@link SyncStreamConfig.SyncStreamConfig}. */
+const JMAP_SYNC_CONFIG = {
+  listPageSize: 50,
+  fetchConcurrency: 5,
+  commitPageSize: 10,
+  maxItemsPerRun: 500,
+} as const satisfies SyncStreamConfig.SyncStreamConfig;
+
+/**
+ * JMAP's {@link MailSyncProvider}: session/account discovery, the email source, the folder→tag map, and
+ * the fused decode+map. Captures {@link JmapMailApi} + {@link Resolver} so the harness never names them.
+ * Mirror of the Gmail provider (`googleMailSyncProvider`).
+ */
+export const jmapMailSyncProvider = (): Layer.Layer<MailSyncProvider, never, JmapMailApi | Resolver> =>
+  Layer.effect(
+    MailSyncProvider,
+    Effect.gen(function* () {
+      // The API is provided into the source stream (leaving `Cursor.Service` for the harness); the full
+      // context into each `process` (whose only needs are API + resolver).
+      const context = yield* Effect.context<JmapMailApi | Resolver>();
+      const providerApi = yield* JmapMailApi;
+      return {
+        name: 'jmap',
+        config: JMAP_SYNC_CONFIG,
+        foreignKeySource: JMAP_DOMAIN,
+        prepare: ({ db, binding, now, token, maxMessages }) =>
+          Effect.gen(function* () {
+            const api = yield* JmapMailApi;
+            const session = yield* api.getSession;
+            const accountId = session.primaryAccounts[MAIL_ACCOUNT_CAPABILITY];
+            if (!accountId) {
+              log.warn('jmap sync: session has no mail account', { username: session.username });
+              return undefined;
+            }
+            const target: JmapMail.Target = { apiUrl: session.apiUrl, accountId, downloadUrl: session.downloadUrl };
+            log('jmap sync: session resolved', { apiUrl: session.apiUrl, accountId });
+
+            // TODO(wittjosiah): Migrate this folder→Tag sync onto a pipeline (source: folders; sink: find-or-create Tag).
+            // Build a folder-id → tag-uri map — mirrors Gmail's `syncLabels`. A well-known role
+            // (`inbox`/`sent`) maps onto the shared canonical system tag; a custom folder gets a
+            // JMAP-scoped provider tag; a dropped role (`archive` — derived as "not in inbox";
+            // `drafts`/`trash`/`junk` — not synced) produces no tag.
+            const { list: folders } = yield* api.mailboxGet(target);
+            const folderTagMap = new Map<string, string>();
+            for (const folder of folders) {
+              const canonical = folder.role ? JMAP_ROLE_TAGS[folder.role] : undefined;
+              if (canonical) {
+                const tag = yield* Effect.promise(() => SystemTags.findOrCreateSystemTag(db, canonical));
+                folderTagMap.set(folder.id, Mailbox.tagUri(tag));
+              } else if (folder.role) {
+                continue;
+              } else {
+                const tag = yield* Effect.promise(() => findOrCreateJmapTag(db, { id: folder.id, name: folder.name }));
+                folderTagMap.set(folder.id, Mailbox.tagUri(tag));
+              }
+            }
+
+            // Keyword → canonical system tag uri (only `$flagged`/starred today). Built once so the
+            // initial map and the reconcile diff resolve the same tags.
+            const keywordTagMap = new Map<string, string>();
+            for (const [keyword, canonical] of Object.entries(JMAP_KEYWORD_TAGS)) {
+              if (!canonical) {
+                continue;
+              }
+              const tag = yield* Effect.promise(() => SystemTags.findOrCreateSystemTag(db, canonical));
+              keywordTagMap.set(keyword, Mailbox.tagUri(tag));
+            }
+
+            // Fused decode + map; `undefined` drops the item (no body, or unmappable). Constructs the
+            // `Change` (an `insert`) directly, so no separate wrapping stage is needed downstream.
+            const toMapped = (
+              email: JmapMail.Email,
+            ): Effect.Effect<EmailStage.Change | undefined, never, JmapMailApi | Resolver> =>
+              Effect.gen(function* () {
+                const decoded = decodeBody(email);
+                if (!decoded) {
+                  return undefined;
+                }
+                const fromAddress = decoded.raw.from?.[0];
+                const contact = fromAddress ? yield* resolve(Person.Person, { email: fromAddress.email }) : undefined;
+                const mapped = mapToMessage(decoded, contact ?? undefined);
+                if (!mapped) {
+                  return undefined;
+                }
+                const folderUris = mapped.mailboxIds.flatMap((folderId) => {
+                  const uri = folderTagMap.get(folderId);
+                  return uri ? [uri] : [];
+                });
+                const keywordUris = mapped.keywords.flatMap((keyword) => {
+                  const uri = keywordTagMap.get(keyword);
+                  return uri ? [uri] : [];
+                });
+                const tagUris = [...folderUris, ...keywordUris];
+                const attachments = yield* fetchAttachments(target, decoded.attachments);
+                return {
+                  _tag: 'insert',
+                  message: mapped.message,
+                  foreignId: decoded.raw.id,
+                  key: new Date(decoded.raw.receivedAt).getTime(),
+                  tagUris,
+                  attachments,
+                } satisfies EmailStage.Change;
+              });
+
+            const toItem = (email: JmapMail.Email): MailSyncItem => ({
+              foreignId: email.id,
+              key: new Date(email.receivedAt).getTime(),
+              process: toMapped(email).pipe(Effect.provide(context)),
+            });
+
+            // The first-tick baseline (and stale-token fallback): the current `Email/get` state with no
+            // delta applied (so mail arriving during backfill is caught by the next incremental, not
+            // missed). Defined once so both call sites share the same capture.
+            const captureFreshDelta = Effect.map(
+              api.emailGet(target, []),
+              (result): DeltaPlan => ({
+                token: result.state,
+                createdIds: undefined,
+                updatedIds: [],
+                hasMoreDelta: false,
+              }),
+            );
+
+            // Resolve the delta plan. An incremental run fetches one bounded `Email/changes` chunk since
+            // the token (`maxChanges` = the per-run budget); `hasMoreChanges` drives `runAgain`, and
+            // `newState` is the chunk boundary the token advances to — so a large delta drains across
+            // runs. A stale token (`cannotCalculateChanges`) falls back to `captureFreshDelta`;
+            // `Effect.catchIf` recovers only that case.
+            const resolveDelta: Effect.Effect<DeltaPlan, JmapApiError, never> =
+              token === undefined
+                ? captureFreshDelta
+                : api.emailChanges(target, token, maxMessages).pipe(
+                    Effect.map((result): DeltaPlan => {
+                      // `updated` are ids whose keywords/mailboxIds may have changed — re-fetched + diffed
+                      // against local tags in the reconcile branch. Exclude ids that are also newly
+                      // created (their full fetch already carries current tags).
+                      const created = new Set(result.created);
+                      const updatedIds = result.updated.filter((id) => !created.has(id));
+                      log('jmap sync: incremental delta', {
+                        created: result.created.length,
+                        updated: result.updated.length,
+                        destroyed: result.destroyed.length,
+                        hasMoreDelta: result.hasMoreChanges,
+                      });
+                      return {
+                        token: result.newState,
+                        createdIds: result.created,
+                        updatedIds,
+                        hasMoreDelta: result.hasMoreChanges,
+                      };
+                    }),
+                    Effect.catchIf(
+                      (error) => error.type === 'cannotCalculateChanges',
+                      () => {
+                        log('jmap sync: state token stale, falling back to window scan');
+                        Cursor.clearToken(binding);
+                        return captureFreshDelta;
+                      },
+                    ),
+                  );
+            const { token: capturedToken, createdIds, updatedIds, hasMoreDelta } = yield* resolveDelta;
+
+            const source: MailSyncSource = {
+              buildSource: ({ windows, filter, tagIndex, onEnumerated, onRetrieved }) => {
+                // Incremental replaces the forward window with the delta's created ids but keeps the
+                // backward backfill window, so each tick still makes backfill progress. When a user filter
+                // is set, the delta's account-wide created ids would bypass it — so fall back to the
+                // filtered forward window scan for additions (the delta still drives reconcile).
+                const forwardIds = filter ? undefined : createdIds;
+                if (forwardIds) {
+                  onEnumerated(forwardIds.length);
+                }
+                return {
+                  additions: jmapEmails(target, folders, {
+                    windows,
+                    filter,
+                    now,
+                    onEnumerated,
+                    onRetrieved,
+                    forwardIds,
+                  }).pipe(
+                    Stream.map(toItem),
+                    Stream.provideService(JmapMailApi, providerApi),
+                    Stream.mapError(MailSyncError.wrap()),
+                  ),
+                  // Empty on non-incremental runs; `jmapReconcile` re-fetches + diffs each `updated` id
+                  // and resolves it to a `Change` itself (it needs the entityId to read local tags).
+                  reconciles: jmapReconcile(updatedIds, target, folderTagMap, keywordTagMap, tagIndex).pipe(
+                    Stream.provideService(JmapMailApi, providerApi),
+                    Stream.mapError(MailSyncError.wrap()),
+                  ),
+                };
+              },
+              nextToken: () => capturedToken,
+              reconcileForeignIds: updatedIds,
+              hasMoreDelta: () => hasMoreDelta,
+            };
+            return source;
+          }).pipe(Effect.provide(context), Effect.mapError(MailSyncError.wrap())),
+      };
+    }),
+  );
+
+/**
+ * Reconcile branch for JMAP: for each `updated` email id, re-fetch its current `mailboxIds` + `keywords`,
+ * map them to Tags, and diff against the message's current *local* tags — a remote-wins, snapshot-free
+ * reconciliation that is idempotent (a crash re-run produces an empty diff). Emits a retag
+ * {@link EmailStage.Change} keyed by the resolved EntityId; ids not in the feed or with no net change are
+ * dropped.
+ *
+ * The two axes differ: **folder** tags are fully remote-wins (added *and* removed, since folder
+ * membership is server-authoritative), while **keyword** tags (starred) are **add-only** — the remote can
+ * add a star but we never auto-remove one, which would clobber a locally-toggled star until we write
+ * local flags back to the provider.
+ */
+const jmapReconcile = (
+  updatedIds: readonly string[],
+  target: JmapMail.Target,
+  folderTagMap: ReadonlyMap<string, string>,
+  keywordTagMap: ReadonlyMap<string, string>,
+  tagIndex: TagIndex.TagIndex,
+): Stream.Stream<EmailStage.Change, JmapApiError, JmapMailApi | Cursor.Service> => {
+  const folderProviderUris = new Set(folderTagMap.values());
+  return Stream.fromIterable(updatedIds).pipe(
+    Stage.map('jmap-reconcile', (id: string) =>
+      Effect.gen(function* () {
+        const { foreignIndex } = yield* Cursor.Service;
+        const entityId = foreignIndex?.get(id);
+        if (!entityId) {
+          return undefined;
+        }
+        const api = yield* JmapMailApi;
+        const { list } = yield* api.emailGet(target, [id]);
+        const email = list[0];
+        if (!email) {
+          return undefined;
+        }
+        const remoteFolderUris = (email.mailboxIds ? Object.keys(email.mailboxIds) : []).flatMap((folderId) => {
+          const uri = folderTagMap.get(folderId);
+          return uri ? [uri] : [];
+        });
+        const remoteKeywordUris = (
+          email.keywords
+            ? Object.entries(email.keywords)
+                .filter(([, set]) => set)
+                .map(([keyword]) => keyword)
+            : []
+        ).flatMap((keyword) => {
+          const uri = keywordTagMap.get(keyword);
+          return uri ? [uri] : [];
+        });
+        const localTags = TagIndex.bind(tagIndex).tags(entityId);
+        const remoteFolders = new Set(remoteFolderUris);
+        // Add any remote folder/keyword tag the message lacks.
+        const addTagIds = [...remoteFolderUris, ...remoteKeywordUris].filter((tagUri) => !localTags.includes(tagUri));
+        // Remove only folder tags the remote no longer has; user tags and starred are never auto-removed.
+        const removeTagIds = localTags.filter((tagUri) => folderProviderUris.has(tagUri) && !remoteFolders.has(tagUri));
+        if (addTagIds.length === 0 && removeTagIds.length === 0) {
+          return undefined;
+        }
+        return { _tag: 'retag', foreignId: id, entityId, addTagIds, removeTagIds } satisfies EmailStage.Change;
+      }),
+    ),
+  );
+};
+
+/**
+ * Downloads each attachment's bytes via `JmapMailApi.downloadBlob`. One failed download (including a
+ * session with no `downloadUrl`) is logged and dropped rather than failing the whole message.
+ */
+const fetchAttachments = (
+  target: JmapMail.Target,
+  attachments: readonly AttachmentMetadata[],
+): Effect.Effect<readonly EmailStage.Attachment[], never, JmapMailApi> =>
+  Effect.gen(function* () {
+    const api = yield* JmapMailApi;
+    const fetched = yield* Effect.forEach(
+      attachments,
+      (attachment) =>
+        api.downloadBlob(target, attachment.blobId, { name: attachment.name, type: attachment.mimeType }).pipe(
+          Effect.map(
+            (bytes): EmailStage.Attachment => ({
+              name: attachment.name,
+              mimeType: attachment.mimeType,
+              size: attachment.size ?? bytes.byteLength,
+              bytes,
+              contentId: attachment.contentId,
+            }),
+          ),
+          Effect.catchAll((error) => {
+            log.catch(error, { blobId: attachment.blobId, name: attachment.name });
+            return Effect.succeed(undefined);
+          }),
+        ),
+      { concurrency: JMAP_SYNC_CONFIG.fetchConcurrency },
+    );
+    return fetched.filter((attachment): attachment is EmailStage.Attachment => attachment !== undefined);
+  });
+
+/**
+ * Streams JMAP email ids over a {@link Cursor.Window}: build the query filter (folder scope + date
+ * bounds + optional user DSL), then paginate. Forward pages oldest-first from `max` so a capped run
+ * advances `max` gap-free instead of jumping to the newest key and stranding the middle; backward pages
+ * newest-first from `min` to the horizon. Backward's upper bound is queried 1ms past `min` so a message
+ * sharing that exact millisecond is re-queried (and deduped) rather than skipped once `min` passes it.
+ * Split from the full-email fetch so a bidirectional run can cap the combined id stream before fetching.
+ */
+const jmapIds = (
+  target: JmapMail.Target,
+  folders: readonly JmapMail.Mailbox[],
+  window: Cursor.Window,
+  options: {
+    filter?: string;
+    /** Reference "now" for the user filter's relative dates (pinned by tests). */
+    now: Date;
+    /** Called with each query page's enumerated id count, to accumulate the retrieval total. */
+    onEnumerated?: (count: number) => void;
+  },
+): Stream.Stream<string, JmapApiError, JmapMailApi> =>
+  Stream.unwrap(
+    Effect.gen(function* () {
+      const api = yield* JmapMailApi;
+
+      const userFilter = options.filter
+        ? JmapMail.parseMailQuery(options.filter, {
+            now: options.now,
+            resolveMailbox: (nameOrRole) => JmapMail.resolveMailboxByNameOrRole(folders, nameOrRole),
+          })
+        : Option.none<Jmap.Filter>();
+      // When the user filter already scopes a mailbox, skip the default folder restriction.
+      const scopesMailbox = Option.match(userFilter, { onNone: () => false, onSome: JmapMail.filterScopesMailbox });
+      // Default to all mail (every folder incl. Sent) so full conversations sync, excluding Trash/Junk/Drafts.
+      const excludedFolderIds = folders
+        .filter((folder) => folder.role === 'trash' || folder.role === 'junk' || folder.role === 'drafts')
+        .map((folder) => folder.id);
+
+      const conditions: Jmap.Filter[] = [];
+      if (!scopesMailbox && excludedFolderIds.length > 0) {
+        conditions.push({ inMailboxOtherThan: excludedFolderIds });
+      }
+      // Bound the query to the window. Backward's `end` is `min` — extend 1ms to include the boundary
+      // millisecond (see doc above).
+      const upperBound = window.direction === 'backward' ? new Date(window.end.getTime() + 1) : window.end;
+      conditions.push({ after: window.start.toISOString() });
+      conditions.push({ before: upperBound.toISOString() });
+      if (Option.isSome(userFilter)) {
+        conditions.push(userFilter.value);
+      }
+      const filter: Jmap.Filter = conditions.length === 1 ? conditions[0] : { operator: 'AND', conditions };
+      log('starting jmap sync', {
+        direction: window.direction,
+        start: window.start.toISOString(),
+        end: window.end.toISOString(),
+        conditions: conditions.length,
+      });
+
+      return Stream.paginateChunkEffect(0, (position: number) =>
+        Effect.gen(function* () {
+          const { ids } = yield* api.emailQuery(target, {
+            filter,
+            sort: [{ property: 'receivedAt', isAscending: window.direction === 'forward' }],
+            position,
+            limit: JMAP_SYNC_CONFIG.listPageSize,
+          });
+          log('jmap sync: queried page', { position, total: ids.length });
+          // Report each page's count as it arrives so the meter's retrieval total leads the fetch.
+          options.onEnumerated?.(ids.length);
+          const next =
+            ids.length < JMAP_SYNC_CONFIG.listPageSize ? Option.none<number>() : Option.some(position + ids.length);
+          return [Chunk.fromIterable(ids), next];
+        }),
+      );
+    }),
+  );
+
+/**
+ * Fetches the full JMAP email for each id. Takes a plain id stream (not a window) so a bidirectional
+ * run can cap the combined stream once, then fetch full emails for exactly the capped set.
+ */
+const jmapEmailsForIds = (
+  target: JmapMail.Target,
+  ids: Stream.Stream<string, JmapApiError, JmapMailApi | Cursor.Service>,
+  options: {
+    /** Called once per email retrieved (full fetch), to advance progress. */
+    onRetrieved?: () => void;
+  } = {},
+): Stream.Stream<JmapMail.Email, JmapApiError, JmapMailApi | Cursor.Service> =>
+  ids.pipe(
+    Stream.flatMap(
+      (id) =>
+        // Drop an id deleted between query and `emailGet` (returns nothing) by filtering out the null.
+        // Do NOT recover the error channel: a real `JmapApiError` must propagate and fail the run so the
+        // durable retry re-fetches, rather than stranding the message once `max` advances.
+        Stream.fromEffect(
+          Effect.gen(function* () {
+            const api = yield* JmapMailApi;
+            const { list } = yield* api.emailGet(target, [id]);
+            options.onRetrieved?.();
+            return list[0];
+          }),
+        ).pipe(Stream.filter(Predicate.isNotNullable)),
+      { concurrency: JMAP_SYNC_CONFIG.fetchConcurrency, bufferSize: 10 },
+    ),
+  );
+
+/**
+ * Streams full JMAP emails for one run: the forward source's ids then the backward window's,
+ * concatenated and fetched in full. Intentionally UNBOUNDED — the harness caps after dedup (see
+ * `runMailSync`). The forward source is either the `max`-anchored forward window (first tick / stale
+ * fallback) or, when `forwardIds` is set, the incremental delta's created ids — in which case the
+ * backward window still runs, so an incremental tick keeps making backfill progress.
+ *
+ * `Cursor.skipCommitted` drops ids already in the dedup set before `emailGet`, so re-queried boundary
+ * ids aren't downloaded; the harness's post-fetch `Cursor.dedupStage` stays the authority.
+ */
+const jmapEmails = (
+  target: JmapMail.Target,
+  folders: readonly JmapMail.Mailbox[],
+  config: {
+    windows: Cursor.Windows;
+    filter?: string;
+    now: Date;
+    onEnumerated?: (count: number) => void;
+    onRetrieved?: () => void;
+    /** Incremental delta's created ids; when set, replaces the forward window (backward still runs). */
+    forwardIds?: readonly string[];
+  },
+): Stream.Stream<JmapMail.Email, JmapApiError, JmapMailApi | Cursor.Service> => {
+  const idsFor = (window: Cursor.Window | undefined) =>
+    window
+      ? jmapIds(target, folders, window, { filter: config.filter, now: config.now, onEnumerated: config.onEnumerated })
+      : Stream.empty;
+  const forward = config.forwardIds ? Stream.fromIterable(config.forwardIds) : idsFor(config.windows.forward);
+
+  return jmapEmailsForIds(
+    target,
+    Stream.concat(forward, idsFor(config.windows.backward)).pipe(Cursor.skipCommitted('skip-committed', (id) => id)),
+    { onRetrieved: config.onRetrieved },
+  );
+};