@bumail/queue 0.13.0 → 0.14.0

package/dist/index.js

@@ -471,6 +471,67 @@ class Events {
   }
 }
 
+// src/sts/settings.ts
+import { MAX_AGE_LIMIT } from "@bumail/mta-sts";
+
+// src/sts/cache.ts
+function createMemoryPolicyCache() {
+  const entries = new Map;
+  return {
+    get: (domain) => entries.get(domain),
+    set(domain, entry) {
+      entries.set(domain, entry);
+    },
+    delete(domain) {
+      entries.delete(domain);
+    }
+  };
+}
+
+// src/sts/settings.ts
+var hasTxt = (value) => typeof value?.txt === "function";
+function stsOf(options, resolver) {
+  if (options === undefined)
+    return;
+  if (typeof options !== "object" || options === null) {
+    throw invalid(`mtaSts must be an object of options, not ${options}`);
+  }
+  const found = options.resolver ?? (hasTxt(resolver) ? resolver : undefined);
+  if (!hasTxt(found)) {
+    throw invalid("mtaSts needs a resolver with txt(name), such as @bumail/dns's nodeResolver(): give mtaSts.resolver, or a resolver that has txt");
+  }
+  const cache = options.cache ?? createMemoryPolicyCache();
+  if (typeof cache.get !== "function" || typeof cache.set !== "function") {
+    throw invalid("mtaSts.cache must be a PolicyCache with get and set");
+  }
+  if (options.fetch !== undefined && typeof options.fetch !== "function") {
+    throw invalid("mtaSts.fetch must be a function");
+  }
+  if (options.skip !== undefined && typeof options.skip !== "function") {
+    throw invalid("mtaSts.skip must be a function");
+  }
+  const { fetchTimeoutMs, maxAgeCap } = options;
+  if (fetchTimeoutMs !== undefined && (typeof fetchTimeoutMs !== "number" || !Number.isFinite(fetchTimeoutMs) || fetchTimeoutMs <= 0 || fetchTimeoutMs > 2147483647)) {
+    throw invalid(`mtaSts.fetchTimeoutMs must be a number of milliseconds above 0 and at most 2147483647, not ${fetchTimeoutMs}`);
+  }
+  if (maxAgeCap !== undefined && (!Number.isSafeInteger(maxAgeCap) || maxAgeCap < 0 || maxAgeCap > MAX_AGE_LIMIT)) {
+    throw invalid(`mtaSts.maxAgeCap must be a whole number of seconds from 0 to ${MAX_AGE_LIMIT}, not ${maxAgeCap}`);
+  }
+  let controller = new AbortController;
+  return {
+    cache,
+    resolver: found,
+    options,
+    inflight: new Map,
+    failures: new Map,
+    signal: () => controller.signal,
+    abort: () => controller.abort(),
+    reset() {
+      controller = new AbortController;
+    }
+  };
+}
+
 // src/queue/settings/numbers.ts
 var MINUTE = 60000;
 var HOUR = 60 * MINUTE;
@@ -690,7 +751,8 @@ function settingsOf(options) {
     now: clock ? () => clock.now() : Date.now,
     send: sender.send,
     close: sender.close,
-    random: options.random ?? Math.random
+    random: options.random ?? Math.random,
+    sts: stsOf(options.mtaSts, options.resolver)
   };
 }
 
@@ -722,7 +784,8 @@ var ERROR_STATUS = {
   TLS_UNAVAILABLE: "7.10",
   TLS_FAILED: "7.0",
   AUTH_UNAVAILABLE: "7.0",
-  INVALID_OPTION: "3.5"
+  INVALID_OPTION: "3.5",
+  MX_NOT_ALLOWED: "7.5"
 };
 function diagnosticOfError(error, temporary, max, host) {
   const text = error instanceof Error ? error.message : String(error);
@@ -787,6 +850,8 @@ function fallsBack(reply, error) {
     return false;
   if (isSmtpError(error) && error.code.startsWith("TLS_"))
     return false;
+  if (isSmtpError(error) && error.tlsResult !== undefined)
+    return false;
   if (GREYLIST.test(reply.text))
     return false;
   if (reply.code === undefined) {
@@ -800,6 +865,320 @@ function fallsBack(reply, error) {
   return REPUTATION.test(reply.text);
 }
 
+// src/queue/session.ts
+import { POLICY_TIMEOUT_MS as POLICY_TIMEOUT_MS3 } from "@bumail/mta-sts";
+
+// src/sts/attempt.ts
+import { matchesMxName } from "@bumail/smtp/client";
+var reportOf = (policy) => policy ? {
+  type: "sts",
+  domain: policy.domain,
+  mode: policy.mode,
+  mx: policy.mx,
+  text: policy.text
+} : { type: "no-policy-found" };
+function tlsEventOf(domain, policy, attempt, at) {
+  const base = {
+    kind: "try",
+    at,
+    domain,
+    policy: reportOf(policy),
+    mxHost: attempt.host,
+    address: attempt.address,
+    ...attempt.localAddress ? { sendingAddress: attempt.localAddress } : {}
+  };
+  const failed = (tlsResult) => ({
+    ...base,
+    ok: false,
+    tlsResult
+  });
+  if (policy && !matchesMxName(attempt.host, policy.mx)) {
+    return failed("validation-failure");
+  }
+  if (attempt.tlsResult)
+    return failed(attempt.tlsResult);
+  if (attempt.tls === false) {
+    if (!attempt.ok)
+      return;
+    return policy ? failed("starttls-not-supported") : { ...base, ok: true };
+  }
+  if (policy && !attempt.tls.verified)
+    return failed("certificate-not-trusted");
+  return { ...base, ok: true };
+}
+
+// src/sts/discover.ts
+import {
+  fetchPolicy,
+  POLICY_TIMEOUT_MS as POLICY_TIMEOUT_MS2,
+  parseStsRecord
+} from "@bumail/mta-sts";
+
+// src/sts/bound.ts
+import { POLICY_TIMEOUT_MS } from "@bumail/mta-sts";
+async function bounded(work, signal, timeoutMs) {
+  if (signal.aborted)
+    return { done: false, why: "aborted" };
+  let timer;
+  let onAbort;
+  const cut = new Promise((resolve) => {
+    timer = setTimeout(() => resolve({ done: false, why: "timeout" }), timeoutMs ?? POLICY_TIMEOUT_MS);
+    onAbort = () => resolve({ done: false, why: "aborted" });
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+  try {
+    return await Promise.race([
+      Promise.resolve(work).then((value) => ({ done: true, value })),
+      cut
+    ]);
+  } finally {
+    clearTimeout(timer);
+    if (onAbort)
+      signal.removeEventListener("abort", onAbort);
+  }
+}
+
+// src/sts/discover.ts
+var FAILED_FETCH_MS = 5 * 60000;
+var active = (domain, policy, text) => policy.mode === "none" ? {} : { policy: { domain, mode: policy.mode, mx: policy.mx, text } };
+var usable = (entry) => {
+  const e = entry;
+  return typeof e === "object" && e !== null && typeof e.id === "string" && typeof e.text === "string" && typeof e.expiresAt === "number" && typeof e.policy === "object" && e.policy !== null && Array.isArray(e.policy.mx) && (e.policy.mode === "enforce" || e.policy.mode === "testing" || e.policy.mode === "none");
+};
+async function cached(sts, domain, now, left) {
+  try {
+    const got = await bounded(sts.cache.get(domain), sts.signal(), left());
+    if (!got.done)
+      return got.why === "aborted" ? { aborted: true } : undefined;
+    if (!usable(got.value))
+      return;
+    if (now < got.value.expiresAt)
+      return got.value;
+    await bounded(sts.cache.delete?.(domain), sts.signal(), left());
+    return;
+  } catch {
+    return;
+  }
+}
+async function recordId(sts, domain, left) {
+  try {
+    const got = await bounded(sts.resolver.txt(`_mta-sts.${domain}`), sts.signal(), left());
+    if (!got.done)
+      return got.why === "aborted" ? { aborted: true } : undefined;
+    const parsed = parseStsRecord(got.value);
+    return parsed.ok ? parsed.value.id : undefined;
+  } catch {
+    return;
+  }
+}
+var timedOut = (ms) => ({
+  code: "TIMEOUT",
+  message: `MTA-STS discovery took more than ${ms} ms`,
+  tlsResult: "sts-policy-fetch-error"
+});
+function remember(sts, key, error, now) {
+  if (sts.failures.size >= 1000) {
+    for (const [k, v] of sts.failures)
+      if (v.until <= now)
+        sts.failures.delete(k);
+    for (const k of sts.failures.keys()) {
+      if (sts.failures.size < 1000)
+        break;
+      sts.failures.delete(k);
+    }
+  }
+  sts.failures.set(key, { error, until: now + FAILED_FETCH_MS });
+}
+async function find(sts, domain, now, fresh) {
+  const total = sts.options.fetchTimeoutMs ?? POLICY_TIMEOUT_MS2;
+  const started = performance.now();
+  const left = () => Math.max(1, total - (performance.now() - started));
+  const kept = await cached(sts, domain, now(), () => Math.min(left(), total / 4));
+  if (kept && "aborted" in kept)
+    return { aborted: true };
+  const keptDiscovery = kept ? active(domain, kept.policy, kept.text) : {};
+  const id = await recordId(sts, domain, left);
+  if (typeof id === "object")
+    return { aborted: true };
+  if (id === undefined)
+    return keptDiscovery;
+  if (kept && kept.id === id && !fresh)
+    return keptDiscovery;
+  const { fetch, maxAgeCap } = sts.options;
+  const failureKey = `${domain}
+${id}`;
+  const failed = fresh ? undefined : sts.failures.get(failureKey);
+  if (failed && now() < failed.until) {
+    return { ...keptDiscovery, fetchError: failed.error };
+  }
+  sts.failures.delete(failureKey);
+  if (performance.now() - started >= total) {
+    return { ...keptDiscovery, fetchError: timedOut(total) };
+  }
+  const budget = left();
+  const result = await fetchPolicy(domain, {
+    signal: sts.signal(),
+    ...fetch ? { fetch } : {},
+    timeoutMs: budget,
+    ...maxAgeCap === undefined ? {} : { maxAgeCap }
+  });
+  if (!result.ok) {
+    if (sts.signal().aborted)
+      return { aborted: true };
+    const shrunk = result.error.code === "TIMEOUT" && budget < total * 0.75;
+    if (!shrunk)
+      remember(sts, failureKey, result.error, now());
+    return { ...keptDiscovery, fetchError: result.error };
+  }
+  const fetchedAt = now();
+  try {
+    await bounded(sts.cache.set(domain, {
+      id,
+      policy: result.policy,
+      text: result.text,
+      fetchedAt,
+      expiresAt: fetchedAt + result.policy.maxAge * 1000
+    }), sts.signal(), total / 4);
+  } catch {}
+  return active(domain, result.policy, result.text);
+}
+function discover(sts, domain, now, fresh) {
+  const key = `${fresh ? "fresh" : "cached"}:${domain}`;
+  const running = sts.inflight.get(key);
+  if (running)
+    return running;
+  const started = find(sts, domain, now, fresh).finally(() => {
+    sts.inflight.delete(key);
+  });
+  sts.inflight.set(key, started);
+  return started;
+}
+
+// src/sts/refusal.ts
+var written = new WeakSet;
+var isPolicyRefusal = (reply) => reply !== undefined && written.has(reply);
+function refusalUpdates(domain, updates, final, why, maxText) {
+  return updates.map((u) => {
+    const code = u.reply?.code;
+    const reply = {
+      ...code !== undefined && code >= 500 === final ? { code } : {},
+      status: final ? "5.7.5" : "4.7.5",
+      text: cleanText(`MTA-STS policy of ${domain} (enforce): ${u.reply?.text ?? why}`, maxText),
+      ...u.reply?.host ? { host: u.reply.host } : {}
+    };
+    written.add(reply);
+    return {
+      address: u.address,
+      status: final ? "failed" : "deferred",
+      reply
+    };
+  });
+}
+
+// src/sts/tally.ts
+var newTally = () => ({
+  tries: [],
+  skipped: [],
+  tlsFailure: false
+});
+function record(tally, domain, policy, attempt, at) {
+  if (attempt.tlsResult)
+    tally.tlsFailure = true;
+  const event = tlsEventOf(domain, policy, attempt, at);
+  if (event)
+    tally.tries.push({ event, delivered: attempt.ok });
+}
+function tell(tally, emit, domain, policy, now, reused) {
+  for (const mxHost of tally.skipped) {
+    emit({
+      kind: "try",
+      at: now(),
+      domain,
+      policy: reportOf(policy),
+      mxHost,
+      ok: false,
+      tlsResult: "validation-failure"
+    });
+  }
+  for (const { event, delivered } of tally.tries) {
+    emit(delivered && reused !== undefined ? { ...event, reused } : event);
+  }
+}
+
+// src/sts/run.ts
+async function prepare(input) {
+  const { sts, domain, emit, maxText, now } = input;
+  let found;
+  try {
+    found = await discover(sts, domain, now, input.refresh);
+  } catch (error) {
+    found = {
+      fetchError: {
+        code: "NETWORK",
+        message: error instanceof Error ? error.message : String(error),
+        tlsResult: "sts-policy-fetch-error"
+      }
+    };
+  }
+  if (found.aborted)
+    return { halt: true };
+  const policy = found.policy;
+  if (found.fetchError) {
+    emit({
+      kind: "policy",
+      at: now(),
+      domain,
+      policy: reportOf(policy),
+      ok: false,
+      tlsResult: found.fetchError.tlsResult
+    });
+  }
+  const enforce = policy?.mode === "enforce" ? policy : undefined;
+  const tally = newTally();
+  const onAttempt = (attempt) => record(tally, domain, policy, attempt, now());
+  const told = (reused) => tell(tally, emit, domain, policy, now, reused);
+  return {
+    options: enforce ? {
+      mxNames: enforce.mx,
+      tls: "required",
+      onAttempt,
+      onSkip: (host) => tally.skipped.push(host)
+    } : { onAttempt },
+    enforcing: enforce !== undefined,
+    delivered: (result) => told(result.reused),
+    failed(error, updates) {
+      const smtp = isSmtpError(error) ? error : undefined;
+      const refusal = enforce && (smtp?.tlsResult !== undefined || tally.tlsFailure && smtp?.temporary !== false);
+      if (smtp?.code === "MX_NOT_ALLOWED") {
+        tally.skipped.push(...smtp.hosts ?? []);
+      }
+      told(undefined);
+      if (!refusal)
+        return { updates, refused: false };
+      const final = input.refresh && now() >= input.giveUpAt;
+      const why = smtp ? smtp.message : "a try broke the policy";
+      return {
+        refused: true,
+        updates: refusalUpdates(domain, updates, final, why, maxText)
+      };
+    }
+  };
+}
+
+// src/queue/retry.ts
+function retryDelay(attempts, retry, random) {
+  const exponent = Math.max(0, attempts - 1);
+  const delay = Math.min(retry.first * retry.factor ** exponent, retry.max);
+  return delay + delay * retry.jitter * Math.min(1, Math.max(0, random()));
+}
+var giveUpAt = (createdAt, retry) => createdAt + retry.giveUpAfter;
+function nextAttemptAt(createdAt, now, attempts, retry, random) {
+  const end = giveUpAt(createdAt, retry);
+  if (now >= end)
+    return;
+  return Math.min(now + retryDelay(attempts, retry, random), end);
+}
+
 // src/queue/route.ts
 function routeOf(settings, domain) {
   return settings.routes.get(domain) ?? settings.options.route ?? "mx";
@@ -863,26 +1242,64 @@ function doubtful(updates, error, key) {
   return updates.map((u) => refused.has(u.address) ? u : marked(u, key));
 }
 var reusedOf = (a) => a.reused === undefined ? {} : { reused: a.reused };
+async function stsFor(ctx, item, domain) {
+  const { settings } = ctx;
+  const { sts } = settings;
+  if (!sts)
+    return;
+  try {
+    if (sts.options.skip?.(item, domain) === true)
+      return;
+  } catch {}
+  const now = settings.now();
+  const end = giveUpAt(item.createdAt, settings.retry);
+  const deadlineMs = (settings.options.deadline ?? 1800) * 1000;
+  const prepared = await prepare({
+    sts,
+    domain,
+    now: settings.now,
+    refresh: now + deadlineMs + (sts.options.fetchTimeoutMs ?? POLICY_TIMEOUT_MS3) >= end,
+    giveUpAt: end,
+    emit: (event) => ctx.events.emit("tls", event),
+    maxText: settings.limits.maxReplyText
+  });
+  return "halt" in prepared ? "halt" : prepared;
+}
 async function session(ctx, item, message, domain, group, route, override) {
   const { settings } = ctx;
   const max = settings.limits.maxReplyText;
+  const sts = route === "mx" && override === undefined ? await stsFor(ctx, item, domain) : undefined;
+  if (sts === "halt") {
+    ctx.halted?.();
+    return { updates: [] };
+  }
   try {
+    const base = sendOptionsOf(settings, domain, item.from, group, override);
     const options = {
-      ...sendOptionsOf(settings, domain, item.from, group, override),
+      ...sts && "resolver" in base ? { ...base, ...sts.options } : base,
       ...dsnOptionsOf(item, group)
     };
     const result = await settings.send(message, options);
+    sts?.delivered(result);
     return {
       updates: outcomesOf(result, group, max),
+      ...sts?.enforcing ? { enforced: true } : {},
       ...result.reused === undefined ? {} : { reused: result.reused }
     };
   } catch (error) {
     if (isRouteError(error))
       ctx.events.emit("error", { error, id: item.id });
-    const updates = outcomesOfError(error, group, max, hostOf(route));
+    const failed = outcomesOfError(error, group, max, hostOf(route));
+    const judged = sts?.failed(error, failed);
+    const updates = judged ? judged.updates : failed;
     const key = route === "mx" ? "mayHaveReachedMx" : "mayHaveReachedFallback";
     const doubt = (route === "mx" || override !== undefined) && isSmtpError(error) && error.afterData;
-    return { updates: doubt ? doubtful(updates, error, key) : updates, error };
+    return {
+      updates: doubt ? doubtful(updates, error, key) : updates,
+      error,
+      ...judged?.refused ? { refused: true } : {},
+      ...sts?.enforcing ? { enforced: true } : {}
+    };
   }
 }
 
@@ -897,7 +1314,7 @@ var stillHeld = (u) => u.status === "failed" ? { ...u, status: "deferred" } : u;
 async function fallBackAtOnce(ctx, item, message, domain, first, fallback, used) {
   const mxHeld = new Set(item.recipients.filter((r) => r.mayHaveReachedMx).map((r) => r.address));
   const mx = new Map(first.updates.filter((u) => u.status !== "delivered" && !u.mayHaveReachedMx && !mxHeld.has(u.address) && fallsBack(u.reply, first.error)).map((u) => [u.address, u]));
-  if (mx.size === 0)
+  if (mx.size === 0 || first.refused || first.enforced)
     return first.updates;
   const second = await session(ctx, item, message, domain, [...mx.keys()], fallback, fallback);
   for (const [address, u] of mx) {
@@ -973,20 +1390,6 @@ async function attemptGroup(ctx, item, message, domain, recipients, used) {
   return [...kept, ...updates, ...bad.map(badAddress)];
 }
 
-// src/queue/retry.ts
-function retryDelay(attempts, retry, random) {
-  const exponent = Math.max(0, attempts - 1);
-  const delay = Math.min(retry.first * retry.factor ** exponent, retry.max);
-  return delay + delay * retry.jitter * Math.min(1, Math.max(0, random()));
-}
-var giveUpAt = (createdAt, retry) => createdAt + retry.giveUpAfter;
-function nextAttemptAt(createdAt, now, attempts, retry, random) {
-  const end = giveUpAt(createdAt, retry);
-  if (now >= end)
-    return;
-  return Math.min(now + retryDelay(attempts, retry, random), end);
-}
-
 // src/queue/notify.ts
 var MAY_HAVE_REACHED = "An earlier attempt may have reached the receiving server, so the message may have been delivered.";
 var MAY_HAVE_REACHED_FALLBACK = "An earlier attempt may have reached the fallback smarthost, so the message may have been delivered.";
@@ -1094,10 +1497,14 @@ async function tellDsns(ctx, item, pending, completed) {
 
 // src/queue/settle.ts
 function expired(update) {
+  const refusal = isPolicyRefusal(update.reply);
+  const status = refusal ? "5.7.5" : "4.4.7";
+  const { code, ...rest } = update.reply ?? {};
+  const reply = refusal && code !== undefined && code < 500 ? { text: "", ...rest, status } : { text: "", ...update.reply, status };
   return {
     address: update.address,
     status: "failed",
-    reply: { text: "", ...update.reply, status: "4.4.7" },
+    reply,
     ...update.mayHaveReachedMx ? { mayHaveReachedMx: true } : {},
     ...update.mayHaveReachedFallback ? { mayHaveReachedFallback: true } : {}
   };
@@ -1197,12 +1604,18 @@ async function deliverItem(ctx, item) {
     if (await store.get(item.id) === undefined)
       return;
     const failed = [...groupsOf(item).values()].flat().map(unreadable);
-    await record(ctx, item, failed, false, undefined, new Map);
+    await record2(ctx, item, failed, false, undefined, new Map);
     return;
   }
   const groups = groupsOf(item);
   let skipped = false;
   const used = new Map;
+  const grouped = {
+    ...ctx,
+    halted: () => {
+      skipped = true;
+    }
+  };
   const outcomes = item.from !== "" && !isMailbox3(item.from) ? [[...groups.values()].flat().map(badSender)] : await Promise.all([...groups].map(async ([domain, group]) => {
     const release = await ctx.domains.acquire(domain);
     try {
@@ -1210,14 +1623,14 @@ async function deliverItem(ctx, item) {
         skipped = true;
         return [];
       }
-      return await attemptGroup(ctx, item, message, domain, group, used);
+      return await attemptGroup(grouped, item, message, domain, group, used);
     } finally {
       release();
     }
   }));
-  await record(ctx, item, outcomes.flat(), skipped, message, used);
+  await record2(ctx, item, outcomes.flat(), skipped, message, used);
 }
-async function record(ctx, item, updates, skipped, message, used) {
+async function record2(ctx, item, updates, skipped, message, used) {
   const { settings, store, events } = ctx;
   const now = settings.now();
   const settled = settle(item, updates, skipped, now, settings);
@@ -1485,12 +1898,14 @@ class Worker {
   async stop() {
     this.#restart = false;
     this.#stopping = true;
+    this.#ctx.settings.sts?.abort();
     this.#wake?.();
     await this.#loop;
     await Promise.all(this.#inFlight);
     try {
       await this.#ctx.settings.close();
     } finally {
+      this.#ctx.settings.sts?.reset();
       this.#loop = undefined;
       this.#stopping = false;
     }
@@ -1540,10 +1955,11 @@ function createQueue(options) {
 export {
   DEFAULT_MAX_RETURN,
   QueueError2 as QueueError,
+  createMemoryPolicyCache,
   createQueue,
   deliveredDsn,
   failedDsn
 };
 
-//# debugId=C2B635B10EF5403B64756E2164756E21
+//# debugId=7BA7A5C5D4F3B31B64756E2164756E21
 //# sourceMappingURL=index.js.map