mailauth 7.1.0 → 7.2.0

package/lib/dmarc/author-domain.js

@@ -3,8 +3,11 @@
 const { formatDomain, toALabel } = require('../tools');
 
 // Characters that can not appear in a domain name taken from an address. Anything like this
-// means the address was malformed and the text after the "@" is not a domain.
-const INVALID_DOMAIN_CHARS = /[\s"<>(),;:\\@[\]]/;
+// means the address was malformed and the text after the "@" is not a domain. Control characters
+// (Cc, including DEL) and invisible format characters (Cf, such as U+200B or the soft hyphen) are
+// not valid in a U-label (RFC 5890 2.3.2.1, RFC 5892), so such a domain has no A-label. U+200C and
+// U+200D are left out, IDNA2008 allows them in some scripts (CONTEXTJ).
+const INVALID_DOMAIN_CHARS = /[\s"<>(),;:\\@[\]\p{Cc}]|[^\P{Cf}\u200c\u200d]/u;
 
 // A domain literal (RFC 5322 3.4.1). It is kept as is, it can not publish a DMARC record.
 const DOMAIN_LITERAL = /^\[[^[\]\\]*\]$/;
@@ -137,17 +140,31 @@ const getAddressDomains = address => {
 // "fromFields" is the number of From header fields the addresses came from, if known. RFC 5322
 // allows only one, and with several the DKIM signature can cover a different field than the one
 // a mail client shows, so their addresses are not merged into one Author Domain.
-const getAuthorDomain = (headerFrom, fromFields) => {
+// "opts.fromSyntax" is the syntax of the From header field (see parseFromHeader). An "invalid"
+// field, or a "lax" one with "opts.strict", gives no Author Domain, as the addresses may not be
+// the ones a reader sees. For backward compatibility "opts" can also be a boolean that is true
+// for a From header field that could not be parsed.
+const getAuthorDomain = (headerFrom, fromFields, opts) => {
+    let fromInvalid = opts;
+    if (opts && typeof opts === 'object') {
+        fromInvalid = opts.fromSyntax === 'invalid' || (!!opts.strict && opts.fromSyntax === 'lax');
+    }
+
     let addresses = [].concat(headerFrom === undefined || headerFrom === null ? [] : headerFrom);
 
     let found = new Map();
     let invalid = false;
+    // a mailbox has a domain part, but it is not a domain name
+    let malformed = !!fromInvalid;
     for (let address of addresses) {
         let entries = getAddressDomains(address);
         if (!entries.length) {
             // a mailbox without a usable domain is not skipped, the domain a reader sees for it
             // is unknown, so the Author Domain can not be determined
             invalid = true;
+            if (typeof address === 'string' && address.indexOf('@') >= 0 && address.slice(address.lastIndexOf('@') + 1).replace(/^\s*\.?\s*$/, '')) {
+                malformed = true;
+            }
         }
         for (let entry of entries) {
             if (!found.has(entry.authorDomain)) {
@@ -162,7 +179,7 @@ const getAuthorDomain = (headerFrom, fromFields) => {
         return { reason: 'multiple-from-fields', domains: Array.from(found.keys()), mailboxes };
     }
 
-    if (invalid && found.size) {
+    if (malformed || (invalid && found.size)) {
         return { reason: 'invalid-author-domain', domains: Array.from(found.keys()), mailboxes };
     }
 
@@ -178,4 +195,252 @@ const getAuthorDomain = (headerFrom, fromFields) => {
     };
 };
 
-module.exports = { getAuthorDomain, getAddressDomains };
+// RFC 5322 3.2.3 atext, with the UTF-8 of RFC 6532 3.2
+const ATEXT = /[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~\u0080-￿]/;
+
+// ASCII control characters other than whitespace. RFC 5322 4.1 only allows them (obs-NO-WS-CTL)
+// in comments, quoted strings and domain literals
+const ASCII_CTL = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
+
+class FromSyntaxError extends Error {}
+
+// Splits a From header field body into atoms, quoted strings, domain literals and single special
+// characters. Whitespace and comments (CFWS) are dropped: where RFC 5322 allows CFWS, including
+// between the atoms of an obs-domain (4.4), it has no meaning, and where it is not allowed, the
+// tokens around it are not valid next to each other anyway.
+const tokenizeFrom = value => {
+    let tokens = [];
+    let i = 0;
+
+    // reads a quoted string, a comment or a domain literal that starts at "i"
+    let readEnclosed = (close, nests) => {
+        let start = i;
+        let depth = 0;
+        for (; i < value.length; i++) {
+            let c = value.charAt(i);
+            if (c === '\\' && close !== ']') {
+                i++;
+            } else if (nests && c === '(') {
+                depth++;
+            } else if (c === close && (!nests || !--depth)) {
+                i++;
+                return value.slice(start, i);
+            } else if (close === ']' && i > start && (c === '[' || c === '\\')) {
+                break;
+            }
+        }
+        throw new FromSyntaxError('unterminated');
+    };
+
+    while (i < value.length) {
+        let c = value.charAt(i);
+        if (c === ' ' || c === '\t' || c === '\r' || c === '\n') {
+            i++;
+        } else if (c === '(') {
+            readEnclosed(')', true);
+        } else if (c === '"') {
+            i++;
+            tokens.push({ type: 'quoted', value: '"' + readEnclosed('"') });
+        } else if (c === '[') {
+            tokens.push({ type: 'literal', value: readEnclosed(']') });
+        } else if (ATEXT.test(c)) {
+            let start = i;
+            while (i < value.length && ATEXT.test(value.charAt(i))) {
+                i++;
+            }
+            tokens.push({ type: 'atom', value: value.slice(start, i) });
+        } else if (ASCII_CTL.test(c)) {
+            throw new FromSyntaxError('control character');
+        } else {
+            // one of the specials ) < > ] : ; @ \ , .
+            tokens.push({ type: c });
+            i++;
+        }
+    }
+
+    return tokens;
+};
+
+// Parses the body of a From header field (RFC 5322 3.6.2 and 4.5.2, a mailbox-list or, by
+// RFC 6854, an address-list) and returns the addr-spec of every mailbox, with CFWS removed and
+// without a source route. "syntax" is "valid", "lax" for a field that only the lenient parsing
+// accepts, or "invalid". An invalid field gives no reliable Author Domain (RFC 9989 4.4), so a
+// domain that is not the one a reader sees can not be evaluated in its place.
+//
+// The lenient parsing accepts:
+//   - a display name without an address, as in "Doe, John <john@example.com>"
+//   - a group that is not closed with ";" at the end of the field
+//   - a domain literal or a leading "." in a display name
+//   - a domain with a trailing dot
+//   - an addr-spec in place of a display name, as in "a@example.com <a@example.com>". Its address
+//     is returned too when it differs from the angle-addr, so that a reader who sees it as the
+//     author does not get the other domain evaluated
+// Anything else that is not RFC 5322 syntax, such as text after an angle-addr or two addresses
+// without a comma between them, makes the field invalid.
+const parseFromHeader = value => {
+    let addresses = [];
+    let lax = false;
+    let tokens;
+    let pos = 0;
+
+    let peek = () => tokens[pos]?.type;
+    let expect = type => {
+        if (peek() !== type) {
+            throw new FromSyntaxError(`expected "${type}"`);
+        }
+        return tokens[pos++];
+    };
+
+    // domain = dot-atom / domain-literal / obs-domain
+    let parseDomain = () => {
+        if (peek() === 'literal') {
+            return tokens[pos++].value;
+        }
+        let domain = expect('atom').value;
+        while (peek() === '.') {
+            pos++;
+            if (peek() !== 'atom') {
+                // a trailing dot, as in "user@example.com."
+                lax = true;
+                return domain + '.';
+            }
+            domain += '.' + tokens[pos++].value;
+        }
+        return domain;
+    };
+
+    // local-part = dot-atom / quoted-string / obs-local-part (word *("." word))
+    let toLocalPart = words => {
+        if (!words.length || words.length % 2 === 0) {
+            throw new FromSyntaxError('invalid local-part');
+        }
+        for (let i = 0; i < words.length; i++) {
+            // words and dots alternate, anything else between two words (such as a domain
+            // literal) is not a local-part
+            let valid = i % 2 === 0 ? words[i].type === 'atom' || words[i].type === 'quoted' : words[i].type === '.';
+            if (!valid) {
+                throw new FromSyntaxError('invalid local-part');
+            }
+        }
+        return words.map(word => (word.type === '.' ? '.' : word.value)).join('');
+    };
+
+    let parseAddrSpec = () => {
+        let words = [];
+        while (['atom', 'quoted', '.'].includes(peek())) {
+            words.push(tokens[pos++]);
+        }
+        let localPart = toLocalPart(words);
+        expect('@');
+        return `${localPart}@${parseDomain()}`;
+    };
+
+    // angle-addr, or obs-angle-addr with a source route that is dropped
+    let parseAngleAddr = () => {
+        expect('<');
+        if (peek() === '@') {
+            // obs-route = obs-domain-list ":"
+            while (peek() === '@' || peek() === ',') {
+                if (tokens[pos++].type === '@') {
+                    parseDomain();
+                }
+            }
+            expect(':');
+        }
+        let address = parseAddrSpec();
+        expect('>');
+        return address;
+    };
+
+    let parseMailbox = inGroup => {
+        // display-name (phrase or obs-phrase), or the local-part of an addr-spec
+        let words = [];
+        while (['atom', 'quoted', '.', 'literal'].includes(peek())) {
+            words.push(tokens[pos++]);
+        }
+
+        let next = peek();
+
+        if (next === '@') {
+            let localPart = toLocalPart(words);
+            pos++;
+            let address = `${localPart}@${parseDomain()}`;
+            if (peek() !== '<') {
+                addresses.push(address);
+                return;
+            }
+            // an addr-spec used as a display name
+            lax = true;
+            let angleAddress = parseAngleAddr();
+            if (address.toLowerCase() !== angleAddress.toLowerCase()) {
+                addresses.push(address);
+            }
+            addresses.push(angleAddress);
+            return;
+        }
+
+        if (words.some(word => word.type === 'literal') || words[0]?.type === '.') {
+            lax = true;
+        }
+
+        if (next === '<') {
+            addresses.push(parseAngleAddr());
+            return;
+        }
+
+        if (next === ':' && !inGroup && words.length) {
+            // group = display-name ":" [group-list] ";" [CFWS]
+            pos++;
+            for (;;) {
+                while (peek() === ',') {
+                    pos++;
+                }
+                if (peek() === ';') {
+                    pos++;
+                    return;
+                }
+                if (!peek()) {
+                    lax = true;
+                    return;
+                }
+                parseMailbox(true);
+                if (peek() !== ',' && peek() !== ';' && peek()) {
+                    throw new FromSyntaxError('unexpected text in group');
+                }
+            }
+        }
+
+        if (!words.length) {
+            throw new FromSyntaxError('unexpected character');
+        }
+
+        // a display name without an address
+        lax = true;
+    };
+
+    try {
+        tokens = tokenizeFrom((value || '').toString());
+        for (;;) {
+            // obs-mbox-list allows empty list elements
+            while (peek() === ',') {
+                pos++;
+            }
+            if (!peek()) {
+                break;
+            }
+            parseMailbox(false);
+            if (peek() && peek() !== ',') {
+                throw new FromSyntaxError('unexpected text after address');
+            }
+        }
+    } catch (err) {
+        if (!(err instanceof FromSyntaxError)) {
+            throw err;
+        }
+        return { addresses, syntax: 'invalid' };
+    }
+
+    return { addresses, syntax: lax ? 'lax' : 'valid' };
+};
+
+module.exports = { getAuthorDomain, getAddressDomains, parseFromHeader };

package/lib/dmarc/get-dmarc-record.js

@@ -102,15 +102,12 @@ const parseDmarcRecord = (txt, opts) => {
     let entries = txt
         .split(';')
         .map(e => e.trim())
-        .filter(e => e)
+        // a fragment without "=" or without a tag name is not a tag, and syntax errors are discarded
+        // (RFC 9989 4.8), so it can not override a valid tag of the same name. The version tag
+        // always has a name and "=", the record matched DMARC_VERSION_RE.
+        .filter(e => e.indexOf('=') > 0)
         .map((e, i) => {
             let splitPos = e.indexOf('=');
-            if (splitPos < 0) {
-                // not a tag, a syntax error that is ignored
-                return [e.toLowerCase().trim(), false];
-            } else if (splitPos === 0) {
-                return [false, e];
-            }
             // whitespace is allowed around "=" (*WSP in the ABNF), so the value must be trimmed
             let name = e.substr(0, splitPos).trim();
             let key = name.toLowerCase();
@@ -156,11 +153,23 @@ const fetchRecord = async (name, resolver, opts) => {
         return false;
     }
 
-    let records;
+    let records = [];
+    // set when a record only starts with the version tag after leading whitespace is removed
+    let trimmed = false;
     try {
         let txt = await resolver(`_dmarc.${name}`, 'TXT');
-        // a custom resolver might return plain strings instead of lists of character-strings
-        records = (txt || []).map(row => (Array.isArray(row) ? row.join('') : String(row)).trim()).filter(row => DMARC_VERSION_RE.test(row));
+        // a custom resolver might return plain strings instead of lists of character-strings.
+        // Trailing whitespace belongs to the last tag value, which is trimmed anyway.
+        for (let row of (txt || []).map(row => (Array.isArray(row) ? row.join('') : String(row)).trimEnd())) {
+            if (DMARC_VERSION_RE.test(row)) {
+                records.push(row);
+            } else if (!opts?.strict && DMARC_VERSION_RE.test(row.trimStart())) {
+                // a record must start with the "v" tag (RFC 9989 4.10), the default mode
+                // also accepts one with whitespace before it
+                records.push(row.trimStart());
+                trimmed = true;
+            }
+        }
     } catch (err) {
         if (NO_RECORD_CODES.has(err.code)) {
             return false;
@@ -169,7 +178,11 @@ const fetchRecord = async (name, resolver, opts) => {
     }
 
     // a record that is not a valid tag-list is discarded like one that is not a DMARC record
-    return (records.length === 1 && parseDmarcRecord(records[0], opts)) || false;
+    let record = (records.length === 1 && parseDmarcRecord(records[0], opts)) || false;
+    if (record && trimmed) {
+        opts?.warnings?.add('record-whitespace');
+    }
+    return record;
 };
 
 // Tree Walks for one message share most of their names, so each name is queried only once.

package/lib/dmarc/verify.js

@@ -22,7 +22,7 @@ const shareParent = (a, b) => a.split('.').pop() === b.split('.').pop();
 const setHidden = (obj, key, value) => Object.defineProperty(obj, key, { value, enumerable: false, configurable: true, writable: true });
 
 const evaluateDmarc = async opts => {
-    let { headerFrom, fromFields, spfDomains, dkimDomains, resolver, arcResult, strict } = opts;
+    let { headerFrom, fromFields, fromSyntax, spfDomains, dkimDomains, resolver, arcResult, strict } = opts;
 
     strict = !!strict;
     resolver = resolver || dns.resolve;
@@ -30,7 +30,8 @@ const evaluateDmarc = async opts => {
     // RFC 9989 5.3.1: several mailboxes are fine if they share one domain, but with zero or more
     // than one Author Domain DMARC validation is not possible. The response stays false, as it
     // always was for these messages, and "reason" says why.
-    const author = getAuthorDomain(headerFrom, fromFields);
+    // a From header field that only the lenient parsing accepts is invalid in strict mode
+    const author = getAuthorDomain(headerFrom, fromFields, { fromSyntax, strict });
     if (!author.authorDomain) {
         return { response: false, reason: author.reason, authorDomains: author.domains };
     }
@@ -41,6 +42,9 @@ const evaluateDmarc = async opts => {
 
     // what the default mode accepted that strict mode would not
     const warnings = new Set();
+    if (fromSyntax === 'lax') {
+        warnings.add('from-syntax');
+    }
 
     let formatResponse = response => {
         if (warnings.size) {
@@ -64,7 +68,7 @@ const evaluateDmarc = async opts => {
         comment: false,
         // ptype properties
         header: {
-            // RFC 8601 2.7.2: header.from is the domain of the RFC5322.From address,
+            // RFC 9989 9.1: header.from is the domain of the RFC5322.From address,
             // not the organizational domain the policy may have been inherited from
             from: domain
         }

package/lib/mailauth.js

@@ -7,7 +7,7 @@ const { dmarc, evaluateDmarc } = require('./dmarc');
 const { arc, createSeal, sealMessage, getARChain, verifyASChain } = require('./arc');
 const { bimi, validateVMC: validateBimiVmc } = require('./bimi');
 const { validateSvg: validateBimiSvg } = require('./bimi/validate-svg');
-const { parseReceived, getClientAddress } = require('./parse-received');
+const { parseReceived, getClientAddress, getClientHelo } = require('./parse-received');
 const { formatDomain, getAlignment, formatAuthservId } = require('./tools');
 const libmime = require('libmime');
 const os = require('node:os');
@@ -60,17 +60,20 @@ const authenticate = async (input, opts) => {
     if (opts.trustReceived) {
         let rcvd = receivedChain?.find(row => row.from?.value);
         if (rcvd) {
-            let helo = rcvd.from.value;
+            let helo = getClientHelo(rcvd);
             // the address of the connection from the TCP-info part of the from clause
             // (RFC 5321 section 4.4), never an address the client claimed in its HELO
-            let ip = getClientAddress(rcvd.from);
+            let ip = getClientAddress(rcvd);
 
-            if (ip && !opts.ip) {
+            // the HELO name of the header belongs to its connection, it is not combined with
+            // an address the caller provided
+            let callerIp = !!opts.ip;
+
+            if (ip && !callerIp) {
                 opts.ip = ip;
             }
 
-            if (helo && !opts.helo && !opts.ip) {
-                // if IP was provided then do not use helo even if it is missing
+            if (helo && !opts.helo && !callerIp) {
                 opts.helo = helo;
             }
 
@@ -135,6 +138,7 @@ const authenticate = async (input, opts) => {
         } = await evaluateDmarc({
             headerFrom: dkimResult.headerFrom,
             fromFields: dkimResult.fromFields,
+            fromSyntax: dkimResult.fromSyntax,
             spfDomains: [].concat((spfResult && spfResult.status.result === 'pass' && spfResult.domain) || []),
             dkimDomains: (dkimResult.results || [])
                 .filter(r => r.status.result === 'pass')
@@ -210,14 +214,13 @@ const authenticate = async (input, opts) => {
         let sealResult = await createSeal(false, {
             headers: dkimResult.headers,
             arc: dkimResult.arc,
-            seal: Object.assign(
-                {
-                    cv: arcResult.status.result,
-                    authResults: arcResult.authResults,
-                    signTime: new Date()
-                },
-                dkimResult.seal
-            ),
+            // the chain status, the AAR and the instance always come from this message, never
+            // from the caller's seal options (RFC 8617 section 5.1 step 4)
+            seal: Object.assign({ signTime: new Date() }, dkimResult.seal, {
+                cv: arcResult.status.result,
+                authResults: arcResult.authResults,
+                i: undefined
+            }),
             strict: opts.strict
         });
 

package/lib/mta-sts.js

@@ -633,11 +633,45 @@ const getPolicy = async (domain, knownPolicy, opts) => {
         return formatResult(Object.assign({}, knownPolicy), 'renewed', warnings);
     }
 
+    // The cached policy that stays in use when the policy for the new ID can not be fetched. A valid
+    // cached "none" policy (or an earlier retry placeholder) enforces nothing, so it is replaced with
+    // a retry placeholder for the new policy ID instead of being reused
+    const canUseCached = () => {
+        if (cacheValid && knownPolicy.mode !== 'none') {
+            return true;
+        }
+        if (!strict && knownPolicy?.id && ['enforce', 'testing'].includes(knownPolicy.mode)) {
+            // lax mode keeps applying an expired policy instead of falling back to no policy
+            warnings.push('expired-cache');
+            return true;
+        }
+        return false;
+    };
+
+    // RFC 8461 3.3: after a failed fetch, further attempts for the same policy ID are limited. The
+    // cached policy that is returned instead records the ID and when it can be fetched again
+    const useCachedAfterFailure = err =>
+        formatResult(
+            Object.assign({}, knownPolicy, {
+                error: err,
+                retryId: policyId,
+                retryAfter: new Date(Date.now() + ERROR_RETRY_DELAY).toISOString()
+            }),
+            'errored',
+            warnings
+        );
+
+    let retryAfter = knownPolicy?.retryId === policyId ? new Date(knownPolicy.retryAfter).getTime() : NaN;
+    if (retryAfter > Date.now() && canUseCached()) {
+        // fetching this policy ID failed recently, keep using the cached policy without a new attempt
+        return formatResult(Object.assign({}, knownPolicy), 'errored', warnings);
+    }
+
     try {
         let policy = await fetchPolicyFile(domain, opts, warnings);
         if (!policy) {
             if (cacheValid) {
-                return useCached(createError(`Policy host mta-sts.${domain} has no address records`, 'policy_host_not_found'));
+                return useCachedAfterFailure(createError(`Policy host mta-sts.${domain} has no address records`, 'policy_host_not_found'));
             }
             return formatResult({ id: false, mode: 'none' }, 'not_found', warnings);
         }
@@ -650,17 +684,9 @@ const getPolicy = async (domain, knownPolicy, opts) => {
             warnings
         );
     } catch (err) {
-        // A valid cached "none" policy (or an earlier retry placeholder) enforces nothing, so it is
-        // replaced with a retry placeholder for the new policy ID below instead of being reused
-        if (cacheValid && knownPolicy.mode !== 'none') {
+        if (canUseCached()) {
             // re-use existing policy on error
-            return useCached(err);
-        }
-
-        if (!strict && knownPolicy?.id && ['enforce', 'testing'].includes(knownPolicy.mode)) {
-            // lax mode keeps applying an expired policy instead of falling back to no policy
-            warnings.push('expired-cache');
-            return useCached(err);
+            return useCachedAfterFailure(err);
         }
 
         // continue as if the domain had no policy, and do not retry fetching this policy ID for a while