@kubun/connector-google-mail 0.17.3 → 0.18.0

package/lib/actions.d.ts

@@ -1,4 +1,4 @@
-import type { ActionHandler, WriteAttachment } from '@kubun/connector';
+import { type ActionHandler, type WriteAttachment } from '@kubun/connector';
 export type GoogleMailActionHandlerConfig = {
     credential: {
         accessToken: string;

package/lib/actions.js

@@ -1,14 +1,16 @@
+import { carryField, refuseCutLists } from '@kubun/connector';
 import { createDraft, createLabel, importMessageAttachment, modifyLabels, sendMessage, updateLabel } from './api.js';
 import { GoogleMailError } from './errors.js';
 // Gmail takes a label color only as the textColor/backgroundColor pair, so a
 // write that changes one of them carries the other from the target document.
 // With no companion to carry (absent or empty), the color cannot be written at all: the write is
-// rejected before any request, rather than sent without the change.
-function withColorPair(params, existing) {
+// rejected before any request, rather than sent without the change. A
+// companion trimmed at sync is refused too (`carryField`), never sent.
+function withColorPair(params, existing, context) {
     const changesOne = 'textColor' in params !== 'backgroundColor' in params;
     if (!changesOne) return params;
     const missing = 'textColor' in params ? 'backgroundColor' : 'textColor';
-    const companion = existing?.[missing];
+    const companion = carryField(existing, missing, context);
     if (typeof companion !== 'string' || companion === '') {
         throw new GoogleMailError(`Google Mail: a label color needs both textColor and backgroundColor, and the label has no ${missing} to keep`, {
             kind: 'rejected'
@@ -58,12 +60,13 @@ export function createGoogleMailActionHandler(config) {
                 sourceFetchHLC: mintActionFetchHLC()
             };
         },
-        async update (model, sourceID, params, existing) {
+        async update (model, sourceID, params, existing, context) {
+            refuseCutLists(params, context);
             if (model === 'GoogleMailLabel') {
                 const record = await updateLabel({
                     accessToken: config.credential.accessToken,
                     labelID: sourceID,
-                    params: withColorPair(params, existing),
+                    params: withColorPair(params, existing, context),
                     fetch: config.fetch
                 });
                 return {

package/lib/api.d.ts

@@ -103,6 +103,17 @@ export declare function getHeader(headers: Array<{
 }> | undefined, name: string): string | undefined;
 export declare function extractEmail(headerValue: string): string;
 export declare function extractEmails(headerValue: string | undefined): Array<string>;
+export type MailParty = {
+    name?: string;
+    email?: string;
+};
+/**
+ * Parses one mailbox out of a header value: `"Alice A" <alice@example.com>`,
+ * `Alice A <alice@example.com>`, or a bare `alice@example.com`. A value that
+ * names no address at all -- malformed or empty -- returns `{}` rather than
+ * a name with no email, or an email this parser only guessed at.
+ */
+export declare function parseMailParty(value: string): MailParty;
 export declare function mapMessageToEntity(message: GmailMessageResource): EntityRecord;
 export declare function getMessage(config: {
     accessToken: string;

package/lib/api.js

@@ -78,6 +78,93 @@ export function extractEmails(headerValue) {
     if (headerValue == null || headerValue.trim() === '') return [];
     return headerValue.split(',').map((part)=>extractEmail(part));
 }
+// A mailbox is either `Display Name <addr@example.com>` (the display name
+// optionally quoted) or a bare `addr@example.com`. This is a liberal but
+// bounded check for "looks like an email address" -- no whitespace, angle
+// bracket or quote characters around a single '@', with at least one '.' in
+// the domain -- never a full RFC 5322 grammar. Gmail headers are already
+// provider-validated, so this only guards against a header this parser
+// cannot make sense of; it never invents an address that is not there.
+const MAILBOX_WITH_NAME = /^([^<>]*)<([^<>]*)>\s*$/;
+const PLAUSIBLE_EMAIL = /^[^\s<>"@]+@[^\s<>"@]+\.[^\s<>"@]+$/;
+function stripQuotes(value) {
+    const trimmed = value.trim();
+    const match = /^"(.*)"$/.exec(trimmed);
+    return match?.[1]?.replace(/\\(["\\])/g, '$1') ?? trimmed;
+}
+/**
+ * Parses one mailbox out of a header value: `"Alice A" <alice@example.com>`,
+ * `Alice A <alice@example.com>`, or a bare `alice@example.com`. A value that
+ * names no address at all -- malformed or empty -- returns `{}` rather than
+ * a name with no email, or an email this parser only guessed at.
+ */ export function parseMailParty(value) {
+    const trimmed = value.trim();
+    if (trimmed === '') return {};
+    const withName = MAILBOX_WITH_NAME.exec(trimmed);
+    if (withName != null) {
+        const email = (withName[2] ?? '').trim();
+        if (!PLAUSIBLE_EMAIL.test(email)) return {};
+        const name = stripQuotes(withName[1] ?? '');
+        return name === '' ? {
+            email
+        } : {
+            name,
+            email
+        };
+    }
+    return PLAUSIBLE_EMAIL.test(trimmed) ? {
+        email: trimmed
+    } : {};
+}
+// Split only at mailbox boundaries outside quoted names and angle addresses.
+// A group label is not a mailbox; its comma-separated members are.
+function splitMailboxes(headerValue) {
+    const parts = [];
+    let current = '';
+    let inQuotes = false;
+    let inAngle = false;
+    let inGroup = false;
+    const push = ()=>{
+        if (current.trim() !== '') parts.push(current.trim());
+        current = '';
+    };
+    for(let i = 0; i < headerValue.length; i++){
+        const char = headerValue[i];
+        if (char === '\\' && inQuotes && i + 1 < headerValue.length) {
+            current += char + headerValue[++i];
+            continue;
+        }
+        if (char === '"' && !inAngle) inQuotes = !inQuotes;
+        if (!inQuotes && char === '<') inAngle = true;
+        if (!inQuotes && char === '>') inAngle = false;
+        if (!inQuotes && !inAngle) {
+            if (char === ':' && !inGroup && /^[^@<>:;]+$/.test(current.trim())) {
+                current = '';
+                inGroup = true;
+                continue;
+            }
+            if (char === ',' || char === ';' && inGroup) {
+                push();
+                if (char === ';') inGroup = false;
+                continue;
+            }
+        }
+        current += char;
+        if (char === '>' && !inQuotes) {
+            const next = headerValue.slice(i + 1).trimStart();
+            if (next.startsWith('<')) push();
+        }
+    }
+    push();
+    return parts;
+}
+function parseMailPartyList(headerValue) {
+    if (headerValue == null || headerValue.trim() === '') return [];
+    return splitMailboxes(headerValue).map(parseMailParty).filter((party)=>party.name != null || party.email != null);
+}
+function nonEmptyPartyList(parties) {
+    return parties.length === 0 ? undefined : parties;
+}
 function findPartByMime(part, mime) {
     if (part.mimeType === mime && part.body?.data != null) return part;
     for (const child of part.parts ?? []){
@@ -117,16 +204,19 @@ export function mapMessageToEntity(message) {
     const headers = message.payload?.headers;
     const labels = message.labelIds ?? [];
     const sentAtMs = message.internalDate != null ? Number.parseInt(message.internalDate, 10) : Number.NaN;
+    const from = parseMailParty(getHeader(headers, 'From') ?? '');
     return {
         sourceService: 'google-mail',
         sourceID: message.id,
         sourceURL: `https://mail.google.com/mail/u/0/#inbox/${message.id}`,
         body: extractBody(message.payload),
-        sender: extractEmail(getHeader(headers, 'From') ?? ''),
+        // A message with no From header, or one this parser cannot make sense
+        // of, carries no `from` rather than an empty Party.
+        from: from.name != null || from.email != null ? from : undefined,
         subject: getHeader(headers, 'Subject'),
         snippet: message.snippet,
-        to: extractEmails(getHeader(headers, 'To')),
-        cc: extractEmails(getHeader(headers, 'Cc')),
+        to: nonEmptyPartyList(parseMailPartyList(getHeader(headers, 'To'))),
+        cc: nonEmptyPartyList(parseMailPartyList(getHeader(headers, 'Cc'))),
         labels,
         unread: labels.includes('UNREAD'),
         sentAt: Number.isNaN(sentAtMs) ? undefined : new Date(sentAtMs).toISOString(),
@@ -262,7 +352,7 @@ function requireString(params, key) {
 }
 // Gmail's write endpoints (messages.send, drafts.create, messages.modify) return only
 // { id, threadId, labelIds } -- no payload/headers/body. Re-fetch the full message so the
-// mapped entity carries real sender/body/subject/etc. instead of empty placeholders.
+// mapped entity carries real from/body/subject/etc. instead of empty placeholders.
 // Everything here runs after the write went out, so any failure is marked sent: the
 // write may have landed, whatever the read-back did.
 async function readBackWrittenMessage(config) {

package/lib/model.d.ts

@@ -5,10 +5,6 @@ export declare const googleMailThreadDefinition: {
     readonly schema: {
         readonly type: "object";
         readonly properties: {
-            readonly title: {
-                readonly type: "string";
-                readonly maxLength: 200;
-            };
             readonly sourceService: {
                 readonly $ref: "#/definitions/SourceService";
             };
@@ -47,7 +43,7 @@ export declare const googleMailThreadDefinition: {
     };
     readonly fieldsMeta: {};
     readonly providerMergePolicy: {
-        readonly group: ["title", "sourceService", "sourceID", "sourceURL"];
+        readonly group: [...string[], "sourceService", "sourceID", "sourceURL"];
     };
 };
 export declare const googleMailMessageDefinition: {
@@ -57,43 +53,16 @@ export declare const googleMailMessageDefinition: {
     readonly schema: {
         readonly type: "object";
         readonly properties: {
-            readonly body: {
-                readonly type: "string";
-            };
-            readonly sender: {
-                readonly type: "string";
-                readonly minLength: 1;
-                readonly maxLength: 320;
-            };
-            readonly subject: {
-                readonly type: "string";
-            };
             readonly snippet: {
                 readonly type: "string";
             };
-            readonly to: {
-                readonly type: "array";
-                readonly title: "To";
-                readonly items: {
-                    readonly type: "string";
-                };
-            };
-            readonly cc: {
-                readonly type: "array";
-                readonly title: "Cc";
-                readonly items: {
-                    readonly type: "string";
-                };
-            };
             readonly labels: {
                 readonly type: "array";
                 readonly title: "Labels";
                 readonly items: {
                     readonly type: "string";
                 };
-            };
-            readonly sentAt: {
-                readonly $ref: "#/definitions/DateTime";
+                readonly maxItems: 100;
             };
             readonly unread: {
                 readonly type: "boolean";
@@ -104,13 +73,11 @@ export declare const googleMailMessageDefinition: {
                 readonly items: {
                     readonly $ref: "#/definitions/Attachment";
                 };
+                readonly maxItems: 100;
             };
             readonly threadID: {
                 readonly type: "string";
             };
-            readonly thread: {
-                readonly $ref: "#/definitions/ThreadDocID";
-            };
             readonly sourceService: {
                 readonly $ref: "#/definitions/SourceService";
             };
@@ -124,14 +91,9 @@ export declare const googleMailMessageDefinition: {
                 readonly $ref: "#/definitions/DateTime";
             };
         };
-        readonly required: ["body", "sender", "threadID", "thread", "sourceService", "sourceID", "lastSyncedAt"];
+        readonly required: ["threadID", "thread", "sourceService", "sourceID", "lastSyncedAt"];
         readonly additionalProperties: false;
         readonly definitions: {
-            readonly ThreadDocID: {
-                readonly type: "string";
-                readonly title: "DocID";
-                readonly pattern: "^k[0-9a-z]{10,120}$";
-            };
             readonly Attachment: {
                 readonly type: "object";
                 readonly title: "Attachment";
@@ -176,14 +138,9 @@ export declare const googleMailMessageDefinition: {
             readonly DateTime: import("json-schema-typed").JSONSchema.String;
         };
     };
-    readonly fieldsMeta: {
-        readonly thread: {
-            readonly type: "document";
-            readonly model: "#0";
-        };
-    };
+    readonly fieldsMeta: {};
     readonly providerMergePolicy: {
-        readonly group: ["body", "sender", "subject", "snippet", "to", "cc", "labels", "sentAt", "unread", "threadID", "thread", "sourceService", "sourceID", "sourceURL"];
+        readonly group: [...string[], "sourceService", "sourceID", "sourceURL"];
     };
 };
 export declare const googleMailLabelDefinition: {

package/lib/model.js

@@ -1,21 +1,25 @@
 import { connectorClusters, connectorModelIDs, externalEntityDefinition } from '@kubun/connector';
-import { modelIDs, modelsClusters } from '@kubun/models';
+import { SHORT_LIST_MAX_ITEMS } from '@kubun/models-core';
+import { messagingClusters, messagingModelDefinitions, messagingModelIDs } from '@kubun/models-messaging';
 import { GraphBuilder } from '@kubun/protocol';
-import { docIDSchema } from '@kubun/scalars';
+const threadKindProviderFields = messagingModelDefinitions.find((definition)=>definition.name === 'ThreadKind')?.providerMergePolicy?.group;
+const messageKindProviderFields = messagingModelDefinitions.find((definition)=>definition.name === 'MessageKind')?.providerMergePolicy?.group;
+const emailKindProviderFields = messagingModelDefinitions.find((definition)=>definition.name === 'EmailKind')?.providerMergePolicy?.group;
+if (threadKindProviderFields == null || messageKindProviderFields == null || emailKindProviderFields == null) {
+    throw new Error('Messaging provider fields are missing');
+}
+// The shared thread field (title) comes from ThreadKind; only what is
+// Google's own is declared here.
 export const googleMailThreadDefinition = {
     name: 'GoogleMailThread',
     behavior: 'default',
     interfaces: [
         connectorModelIDs.ExternalEntity,
-        modelIDs.ThreadKind
+        messagingModelIDs.ThreadKind
     ],
     schema: {
         type: 'object',
         properties: {
-            title: {
-                type: 'string',
-                maxLength: 200
-            },
             sourceService: {
                 $ref: '#/definitions/SourceService'
             },
@@ -44,60 +48,35 @@ export const googleMailThreadDefinition = {
     // provider content, so it stays outside the group.
     providerMergePolicy: {
         group: [
-            'title',
+            ...threadKindProviderFields,
             'sourceService',
             'sourceID',
             'sourceURL'
         ]
     }
 };
+// Generic message fields come from MessageKind, and mail fields from EmailKind.
+// Only what is Google's own is declared here.
 export const googleMailMessageDefinition = {
     name: 'GoogleMailMessage',
     behavior: 'default',
     interfaces: [
         connectorModelIDs.ExternalEntity,
-        modelIDs.MessageKind
+        messagingModelIDs.EmailKind
     ],
     schema: {
         type: 'object',
         properties: {
-            body: {
-                type: 'string'
-            },
-            sender: {
-                type: 'string',
-                minLength: 1,
-                maxLength: 320
-            },
-            subject: {
-                type: 'string'
-            },
             snippet: {
                 type: 'string'
             },
-            to: {
-                type: 'array',
-                title: 'To',
-                items: {
-                    type: 'string'
-                }
-            },
-            cc: {
-                type: 'array',
-                title: 'Cc',
-                items: {
-                    type: 'string'
-                }
-            },
             labels: {
                 type: 'array',
                 title: 'Labels',
                 items: {
                     type: 'string'
-                }
-            },
-            sentAt: {
-                $ref: '#/definitions/DateTime'
+                },
+                maxItems: SHORT_LIST_MAX_ITEMS
             },
             unread: {
                 type: 'boolean'
@@ -107,14 +86,12 @@ export const googleMailMessageDefinition = {
                 title: 'Attachments',
                 items: {
                     $ref: '#/definitions/Attachment'
-                }
+                },
+                maxItems: SHORT_LIST_MAX_ITEMS
             },
             threadID: {
                 type: 'string'
             },
-            thread: {
-                $ref: '#/definitions/ThreadDocID'
-            },
             sourceService: {
                 $ref: '#/definitions/SourceService'
             },
@@ -129,8 +106,6 @@ export const googleMailMessageDefinition = {
             }
         },
         required: [
-            'body',
-            'sender',
             'threadID',
             'thread',
             'sourceService',
@@ -140,7 +115,6 @@ export const googleMailMessageDefinition = {
         additionalProperties: false,
         definitions: {
             ...externalEntityDefinition.schema.definitions,
-            ThreadDocID: docIDSchema,
             Attachment: {
                 type: 'object',
                 title: 'Attachment',
@@ -174,29 +148,22 @@ export const googleMailMessageDefinition = {
             }
         }
     },
-    fieldsMeta: {
-        thread: {
-            type: 'document',
-            model: '#0'
-        }
-    },
+    fieldsMeta: {},
     // lastSyncedAt is device-local sync bookkeeping; attachments carries the
     // locally-materialized `blob` annotation (absent until an explicit import
     // fills it), so the whole field stays outside the group rather than
     // splitting a single top-level field between provider and device content.
     providerMergePolicy: {
         group: [
-            'body',
-            'sender',
-            'subject',
+            ...messageKindProviderFields.slice(0, 2),
+            ...emailKindProviderFields.slice(0, 1),
             'snippet',
-            'to',
-            'cc',
+            ...emailKindProviderFields.slice(1),
             'labels',
-            'sentAt',
+            ...messageKindProviderFields.slice(2, 3),
             'unread',
             'threadID',
-            'thread',
+            ...messageKindProviderFields.slice(3),
             'sourceService',
             'sourceID',
             'sourceURL'
@@ -281,7 +248,7 @@ export const googleMailLabelDefinition = {
 const builder = new GraphBuilder({
     externalClusters: {
         ...connectorClusters,
-        ...modelsClusters
+        ...messagingClusters
     }
 });
 builder.add(googleMailThreadDefinition); // #0

package/lib/provider.js

@@ -1,6 +1,26 @@
 import { fetchOrAbsent, parseMaxAgeDays, resolveModelID } from '@kubun/connector';
+import { mapConcurrent, retry } from '@sozai/async';
 import { buildHistoryListURL, buildLabelsListURL, buildMessagesListURL, buildProfileURL, getHeader, getLabel, getMessage, getThread, mailAPIClient, mapLabelToEntity, mapMessageToEntity, mapThreadToEntity } from './api.js';
 import { googleMailClusters } from './model.js';
+// messages.get costs 5 quota units against Gmail's 250 units/s per-user limit.
+const MESSAGE_FETCH_CONCURRENCY = 4;
+// A burst can still trip Gmail's per-user rate limit; a limited message is
+// retried on its own so the rest of the sync keeps going.
+const MESSAGE_RATE_LIMIT_POLICY = {
+    maxAttempts: 6,
+    backoff: {
+        initialMs: 250,
+        multiplier: 2,
+        maxMs: 8000,
+        jitter: true
+    }
+};
+// 429, or Gmail's 403 rateLimitExceeded / userRateLimitExceeded.
+function isRateLimited(error) {
+    const { httpStatus, message } = error ?? {};
+    if (httpStatus === 429) return true;
+    return httpStatus === 403 && typeof message === 'string' && /rate ?limit/i.test(message);
+}
 export const DEFAULT_QUERY = 'in:inbox';
 const THREAD_MODEL_ID = resolveModelID(googleMailClusters, 'GoogleMailThread');
 const MESSAGE_MODEL_ID = resolveModelID(googleMailClusters, 'GoogleMailMessage');
@@ -51,10 +71,20 @@ export function createGoogleMailProvider(config) {
     // shared api.ts getMessage so both the bulk and targeted (fetchEntity)
     // paths share one 404-tagging choke point.
     async function getMessageResource(id, signal) {
-        return getMessage({
-            accessToken: config.credential.accessToken,
-            messageID: id,
-            fetch: fetchFn,
+        return retry(({ signal: attemptSignal })=>getMessage({
+                accessToken: config.credential.accessToken,
+                messageID: id,
+                fetch: fetchFn,
+                signal: attemptSignal
+            }), {
+            policy: MESSAGE_RATE_LIMIT_POLICY,
+            signal,
+            retryable: isRateLimited
+        });
+    }
+    function getMessageResources(ids, signal) {
+        return mapConcurrent(ids, (id, _index, callSignal)=>getMessageResource(id, callSignal), {
+            limit: MESSAGE_FETCH_CONCURRENCY,
             signal
         });
     }
@@ -90,10 +120,7 @@ export function createGoogleMailProvider(config) {
         const windowStart = windowStartSeconds(params.boundary, prior?.windowStart);
         const query = applyAfter(config.query ?? DEFAULT_QUERY, windowStart);
         const ids = await listMessageIDs(query, params.signal);
-        const messages = [];
-        for (const { id } of ids){
-            messages.push(await getMessageResource(id, params.signal));
-        }
+        const messages = await getMessageResources(ids.map(({ id })=>id), params.signal);
         // Thread batch: one thread per distinct threadId, titled by its first message's subject.
         const threadSubjects = new Map();
         for (const m of messages){
@@ -176,10 +203,7 @@ export function createGoogleMailProvider(config) {
         const survivingIDs = [
             ...addedIDs.keys()
         ].filter((id)=>!deletedIDs.has(id));
-        const messages = [];
-        for (const id of survivingIDs){
-            messages.push(await getMessageResource(id, params.signal));
-        }
+        const messages = await getMessageResources(survivingIDs, params.signal);
         const threadSubjects = new Map();
         for (const m of messages){
             if (!threadSubjects.has(m.threadId)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/connector-google-mail",
-  "version": "0.17.3",
+  "version": "0.18.0",
   "keywords": [],
   "license": "see LICENSE.md",
   "sideEffects": false,
@@ -15,14 +15,16 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "@kubun/connector": "^0.17.3",
-    "@kubun/credential-types": "^0.17.2",
-    "@kubun/hlc": "^0.17.0",
-    "@kubun/models": "^0.17.0",
-    "@kubun/protocol": "^0.17.2",
-    "@kubun/provider-google": "^0.17.1",
-    "@kubun/scalars": "^0.17.0",
-    "@sozai/codec": "^0.4.0"
+    "@kubun/connector": "^0.18.0",
+    "@kubun/credential-types": "^0.18.0",
+    "@kubun/hlc": "^0.18.0",
+    "@kubun/models-core": "^0.18.0",
+    "@kubun/models-messaging": "^0.18.0",
+    "@kubun/protocol": "^0.18.0",
+    "@kubun/provider-google": "^0.18.0",
+    "@kubun/scalars": "^0.18.0",
+    "@sozai/async": "^0.4.1",
+    "@sozai/codec": "^0.4.3"
   },
   "publishConfig": {
     "access": "public"