backend-manager 5.2.2 → 5.2.3

package/CHANGELOG.md

@@ -14,6 +14,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+# [5.2.3] - 2026-05-22
+
+### Added
+
+- **`marketing.sendgrid.listId` in `templates/backend-manager-config.json`.** Empty-string placeholder for OMEGA's `sendgrid/ensure/list.js` to populate at brand-onboarding time. Mirrors the existing `marketing.beehiiv.publicationId` convention.
+- **`_test.*` local-part block** in `src/manager/libraries/email/data/blocked-local-patterns.js`. Test-suite accounts (`_test.<scenario>@somiibo.com`) are now blocked from reaching SendGrid + Beehiiv. The carved-out exception is `_test.allow_*` — used for live-provider integration tests that intentionally need to round-trip a real contact.
+
+### Changed
+
+- **`Marketing.add()` and `Marketing.sync()` now use the full `validate()` pipeline** instead of just `isCorporate()`. Single SSOT for "is this a valid marketing email" — runs format → disposable → corporate → localPart in one call. Stricter behavior: disposable-domain emails (mailinator etc.) and junk local-parts (`noreply`, `test*`, `_test.*`) are now blocked from marketing lists. They were previously waved through because the gate only checked corporate domains.
+- **SendGrid list-ID lookup is now config-only.** `src/manager/libraries/email/providers/sendgrid.js#getListId()` reads `Manager.config.marketing.sendgrid.listId` and returns null if missing — no more runtime API call, no more fuzzy-match-by-brand-name. Old fuzzy logic kept commented out as `getListIdByFuzzyMatch()` backstop. **Brands must run OMEGA's sendgrid service to populate `listId` before this version sees their list assignments work** (without it, contacts land in SendGrid's global "All Contacts" pool, not the brand list).
+- **Beehiiv publication-ID lookup is now config-only.** `src/manager/libraries/email/providers/beehiiv.js#getPublicationId()` reads `Manager.config.marketing.beehiiv.publicationId` and returns null if missing — same shape as SendGrid above. Old fuzzy logic kept commented out as backstop. Beehiiv side already preferred config when set, but now there's no API fallback.
+- **`marketing.beehiiv.publicationId` is now an always-present empty string in the config template** (was a commented-out hint). Matches the SendGrid `listId` shape and means `Manager.config.marketing.beehiiv.publicationId` always returns `""` (never `undefined`) for legacy brands.
+- **SendGrid API timeouts bumped from 10s → 60s** via a new top-level `SENDGRID_TIMEOUT_MS` constant in `sendgrid.js`. All 9 fetch sites updated. Catches the intermittent SendGrid backend hiccups that were dropping signups silently with "Request timed out".
+
 # [5.2.2] - 2026-05-21
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend-manager",
-  "version": "5.2.2",
+  "version": "5.2.3",
   "description": "Quick tools for developing Firebase functions",
   "main": "src/manager/index.js",
   "bin": {

package/src/manager/libraries/email/data/blocked-local-patterns.js

@@ -3,9 +3,14 @@
  * Kept as JS (not JSON) so patterns stay as native RegExp literals.
  */
 module.exports = [
-  /^\d+$/,           // All numeric: 123456
-  /^(.)\1{2,}$/,     // Repeating single char: aaaa, xxxx
-  /^[a-z]{1,2}\d+$/, // Single letter + numbers: a123, x999
-  /^test/,           // Starts with test: test, testuser, test123, test.user
-  /^example/,        // Starts with example: example, exampleuser, example.user
+  /^\d+$/,                // All numeric: 123456
+  /^(.)\1{2,}$/,          // Repeating single char: aaaa, xxxx
+  /^[a-z]{1,2}\d+$/,      // Single letter + numbers: a123, x999
+  /^test/,                // Starts with test: test, testuser, test123, test.user
+  /^example/,             // Starts with example: example, exampleuser, example.user
+  // Test-suite accounts use `_test.<scenario>@...` so they don't collide with
+  // real signups. Block them from reaching SendGrid/Beehiiv so live lists stay
+  // clean. `_test.allow_*` is the carved-out exception for live-provider
+  // integration tests that intentionally need to round-trip a real contact.
+  /^_test\.(?!allow_)/,
 ];

package/src/manager/libraries/email/marketing/index.js

@@ -29,7 +29,7 @@ const md = new MarkdownIt({ html: true, breaks: true, linkify: true });
 
 const { TEMPLATES, GROUPS, SENDERS } = require('../constants.js');
 const { tagLinks } = require('../utm.js');
-const { isCorporate } = require('../validation.js');
+const { validate } = require('../validation.js');
 const sendgridProvider = require('../providers/sendgrid.js');
 const beehiivProvider = require('../providers/beehiiv.js');
 
@@ -73,9 +73,13 @@ Marketing.prototype.add = async function (options) {
     return {};
   }
 
-  if (isCorporate(email)) {
-    assistant.warn(`Marketing.add(): Blocked corporate/social-media domain, skipping: ${email}`);
-    return { blocked: 'corporate', email };
+  // SSOT for what "valid marketing email" means — format, disposable, corporate,
+  // localPart junk (incl. _test.* test-suite accounts). Single validate() call
+  // catches all of these before we hit SendGrid/Beehiiv.
+  const validation = await validate(email);
+  if (!validation.valid) {
+    assistant.warn(`Marketing.add(): Validation failed, skipping: ${email}`, validation.checks);
+    return { blocked: 'validation', email, checks: validation.checks };
   }
 
   if (assistant.isTesting() && !process.env.TEST_EXTENDED_MODE) {
@@ -157,9 +161,13 @@ Marketing.prototype.sync = async function (userDocOrUid) {
     return {};
   }
 
-  if (isCorporate(email)) {
-    assistant.warn(`Marketing.sync(): Blocked corporate/social-media domain, skipping: ${email}`);
-    return { blocked: 'corporate', email };
+  // SSOT for what "valid marketing email" means — format, disposable, corporate,
+  // localPart junk (incl. _test.* test-suite accounts). Single validate() call
+  // catches all of these before we hit SendGrid/Beehiiv.
+  const validation = await validate(email);
+  if (!validation.valid) {
+    assistant.warn(`Marketing.sync(): Validation failed, skipping: ${email}`, validation.checks);
+    return { blocked: 'validation', email, checks: validation.checks };
   }
 
   if (assistant.isTesting() && !process.env.TEST_EXTENDED_MODE) {

package/src/manager/libraries/email/providers/beehiiv.js

@@ -131,80 +131,96 @@ async function removeSubscriber(email, publicationId) {
 }
 
 /**
- * Get a Beehiiv publication ID by brand name (fuzzy match).
+ * Get this brand's Beehiiv publication ID.
  *
- * @param {string} brandName
- * @returns {string|null} Publication ID or null
+ * Reads `Manager.config.marketing.beehiiv.publicationId` — populated by
+ * OMEGA's `beehiiv/ensure/publication.js` at brand-onboarding time. No
+ * runtime API call, no fuzzy-match fragility.
+ *
+ * If the brand hasn't been onboarded yet (publicationId missing/empty), logs
+ * a warning and returns null — the marketing sync will skip Beehiiv for this
+ * brand. Fix: run OMEGA's beehiiv service to populate.
+ *
+ * @returns {string|null} Publication ID or null if not configured
  */
-let _publicationIdCache = null;
-
-async function getPublicationId() {
-  if (_publicationIdCache) {
-    return _publicationIdCache;
-  }
-
-  // Use publicationId from config if set (skips API call)
-  const configPubId = Manager.config?.marketing?.beehiiv?.publicationId;
-
-  if (configPubId) {
-    _publicationIdCache = configPubId;
-    return configPubId;
-  }
-
-  // Fuzzy-match by brand name (guard against uninitialized Manager singleton —
-  // happens in test stubs that build their own Manager without init()).
-  const brandName = Manager.config?.brand?.name;
+function getPublicationId() {
+  const publicationId = Manager.config?.marketing?.beehiiv?.publicationId;
 
-  if (!brandName) {
-    console.error('Beehiiv: Brand name is required to find publication');
+  if (!publicationId) {
+    console.warn(
+      'Beehiiv: marketing.beehiiv.publicationId is not set in config. '
+      + 'Subscriber will NOT be added to a publication. '
+      + 'Run OMEGA to populate.',
+    );
     return null;
   }
 
-  const brandNameLower = brandName.toLowerCase();
-  const allPublications = [];
-  let page = 1;
-  const limit = 100;
-
-  try {
-    while (true) {
-      const data = await fetch(`${BASE_URL}/publications?limit=${limit}&page=${page}`, {
-        response: 'json',
-        headers: headers(),
-        timeout: 10000,
-      });
-
-      if (!data.data || data.data.length === 0) {
-        break;
-      }
-
-      const matchedPub = data.data.find(pub =>
-        pub.name.toLowerCase() === brandNameLower
-        || pub.name.toLowerCase().includes(brandNameLower)
-        || brandNameLower.includes(pub.name.toLowerCase())
-      );
-
-      if (matchedPub) {
-        _publicationIdCache = matchedPub.id;
-        return matchedPub.id;
-      }
-
-      allPublications.push(...data.data);
-
-      if (data.data.length < limit) {
-        break;
-      }
-
-      page++;
-    }
-
-    console.error(`Beehiiv: No publication matched brand "${brandName}". Available: ${allPublications.map(p => p.name).join(', ')}`);
-  } catch (e) {
-    console.error('Beehiiv publication lookup error:', e);
-  }
-
-  return null;
+  return publicationId;
 }
 
+// LEGACY: Fuzzy-match-by-brand-name fallback. Kept commented out as a backstop
+// in case the config-based approach has an edge case we haven't seen yet.
+// Delete once we've verified the config-based approach works across all brands.
+//
+// let _publicationIdCache = null;
+//
+// async function getPublicationIdByFuzzyMatch() {
+//   if (_publicationIdCache) {
+//     return _publicationIdCache;
+//   }
+//
+//   const brandName = Manager.config?.brand?.name;
+//
+//   if (!brandName) {
+//     console.error('Beehiiv: Brand name is required to find publication');
+//     return null;
+//   }
+//
+//   const brandNameLower = brandName.toLowerCase();
+//   const allPublications = [];
+//   let page = 1;
+//   const limit = 100;
+//
+//   try {
+//     while (true) {
+//       const data = await fetch(`${BASE_URL}/publications?limit=${limit}&page=${page}`, {
+//         response: 'json',
+//         headers: headers(),
+//         timeout: 10000,
+//       });
+//
+//       if (!data.data || data.data.length === 0) {
+//         break;
+//       }
+//
+//       const matchedPub = data.data.find(pub =>
+//         pub.name.toLowerCase() === brandNameLower
+//         || pub.name.toLowerCase().includes(brandNameLower)
+//         || brandNameLower.includes(pub.name.toLowerCase())
+//       );
+//
+//       if (matchedPub) {
+//         _publicationIdCache = matchedPub.id;
+//         return matchedPub.id;
+//       }
+//
+//       allPublications.push(...data.data);
+//
+//       if (data.data.length < limit) {
+//         break;
+//       }
+//
+//       page++;
+//     }
+//
+//     console.error(`Beehiiv: No publication matched brand "${brandName}". Available: ${allPublications.map(p => p.name).join(', ')}`);
+//   } catch (e) {
+//     console.error('Beehiiv publication lookup error:', e);
+//   }
+//
+//   return null;
+// }
+
 /**
  * Add a contact to Beehiiv — resolves publication, adds subscriber with optional custom fields.
  *
@@ -217,7 +233,7 @@ async function getPublicationId() {
  * @returns {{ success: boolean, id?: string, error?: string }}
  */
 async function addContact({ email, firstName, lastName, company, source, customFields }) {
-  const publicationId = await getPublicationId();
+  const publicationId = getPublicationId();
 
   if (!publicationId) {
     return { success: false, error: 'Publication not found' };
@@ -245,7 +261,7 @@ async function addContact({ email, firstName, lastName, company, source, customF
  * @returns {{ success: boolean, deleted?: boolean, skipped?: boolean, error?: string }}
  */
 async function removeContact(email) {
-  const publicationId = await getPublicationId();
+  const publicationId = getPublicationId();
 
   if (!publicationId) {
     return { success: false, error: 'Publication not found' };
@@ -290,7 +306,7 @@ async function resolveSegmentIds() {
     return _segmentIdCache;
   }
 
-  const publicationId = await getPublicationId();
+  const publicationId = getPublicationId();
 
   if (!publicationId) {
     return {};
@@ -337,7 +353,7 @@ async function resolveSegmentIds() {
  * @returns {{ success: boolean, id?: string, scheduled?: boolean, error?: string }}
  */
 async function createPost(options) {
-  const publicationId = options.publicationId || await getPublicationId();
+  const publicationId = options.publicationId || getPublicationId();
 
   if (!publicationId) {
     return { success: false, error: 'Publication not found' };

package/src/manager/libraries/email/providers/sendgrid.js

@@ -9,6 +9,12 @@ const { resolveFieldValues } = require('../constants.js');
 
 const BASE_URL = 'https://api.sendgrid.com/v3';
 
+// SendGrid's API is normally fast (<2s) but spikes past 10s during their
+// hiccups, dropping signups silently. 60s is generous but harmless — the
+// metadata calls (resolveFieldIds, getListId) are cached for the process
+// lifetime so a slow first call costs nothing in steady state.
+const SENDGRID_TIMEOUT_MS = 60000;
+
 // --- Internal helpers ---
 
 function headers() {
@@ -35,7 +41,7 @@ async function resolveFieldIds() {
     const data = await fetch(`${BASE_URL}/marketing/field_definitions`, {
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     _fieldIdCache = {};
@@ -70,7 +76,7 @@ async function resolveSegmentIds() {
     const data = await fetch(`${BASE_URL}/marketing/segments/2.0`, {
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     _segmentIdCache = {};
@@ -137,7 +143,7 @@ async function removeContact(email) {
       method: 'post',
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
       body: { emails: [email] },
     });
 
@@ -152,7 +158,7 @@ async function removeContact(email) {
       method: 'delete',
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     if (deleteData.job_id) {
@@ -167,69 +173,96 @@ async function removeContact(email) {
 }
 
 /**
- * Get a SendGrid list ID by brand name (fuzzy match).
+ * Get this brand's SendGrid Marketing list ID.
+ *
+ * Reads `Manager.config.marketing.sendgrid.listId` — populated by OMEGA's
+ * `sendgrid/ensure/list.js` at brand-onboarding time, same as how Beehiiv's
+ * `publicationId` works. No runtime API call, no fuzzy-match fragility.
  *
- * @param {string} brandName
- * @returns {string|null} List ID or null
+ * If the brand hasn't been onboarded yet (listId missing/empty), logs a
+ * warning and returns null — the marketing sync will still succeed, but the
+ * contact lands in SendGrid's global pool instead of the brand's list. Fix:
+ * run OMEGA's sendgrid service to populate the config.
+ *
+ * @returns {string|null} List ID or null if not configured
  */
-async function getListId() {
-  const brandName = Manager.config.brand?.name;
-  const brandNameLower = (brandName || '').toLowerCase();
-  const allLists = [];
-  let pageToken = '';
-  const pageSize = 1000;
-
-  try {
-    while (true) {
-      const url = `${BASE_URL}/marketing/lists?page_size=${pageSize}${pageToken ? `&page_token=${pageToken}` : ''}`;
-      const data = await fetch(url, {
-        response: 'json',
-        headers: headers(),
-        timeout: 10000,
-      });
-
-      if (!data.result || data.result.length === 0) {
-        break;
-      }
-
-      const matchedList = data.result.find(list =>
-        list.name.toLowerCase() === brandNameLower
-        || list.name.toLowerCase().includes(brandNameLower)
-        || brandNameLower.includes(list.name.toLowerCase())
-      );
-
-      if (matchedList) {
-        return matchedList.id;
-      }
-
-      allLists.push(...data.result);
-
-      if (!data._metadata?.next) {
-        break;
-      }
-
-      const nextUrl = new URL(data._metadata.next);
-      pageToken = nextUrl.searchParams.get('page_token');
-
-      if (!pageToken) {
-        break;
-      }
-    }
-
-    if (allLists.length === 1) {
-      return allLists[0].id;
-    }
-
-    if (allLists.length > 0) {
-      console.error(`SendGrid: No list matched brand "${brandName}". Available: ${allLists.map(l => l.name).join(', ')}`);
-    }
-  } catch (e) {
-    console.error('SendGrid list lookup error:', e);
+function getListId() {
+  const listId = Manager.config.marketing?.sendgrid?.listId;
+
+  if (!listId) {
+    console.warn(
+      'SendGrid: marketing.sendgrid.listId is not set in config. '
+      + 'Contact will be added to All Contacts only, not the brand list. '
+      + 'Run OMEGA to populate.',
+    );
+    return null;
   }
 
-  return null;
+  return listId;
 }
 
+// LEGACY: Fuzzy-match-by-brand-name fallback. Kept commented out as a backstop
+// in case the config-based approach has an edge case we haven't seen yet.
+// Delete once we've verified the config-based approach works across all brands.
+//
+// async function getListIdByFuzzyMatch() {
+//   const brandName = Manager.config.brand?.name;
+//   const brandNameLower = (brandName || '').toLowerCase();
+//   const allLists = [];
+//   let pageToken = '';
+//   const pageSize = 1000;
+//
+//   try {
+//     while (true) {
+//       const url = `${BASE_URL}/marketing/lists?page_size=${pageSize}${pageToken ? `&page_token=${pageToken}` : ''}`;
+//       const data = await fetch(url, {
+//         response: 'json',
+//         headers: headers(),
+//         timeout: SENDGRID_TIMEOUT_MS,
+//       });
+//
+//       if (!data.result || data.result.length === 0) {
+//         break;
+//       }
+//
+//       const matchedList = data.result.find(list =>
+//         list.name.toLowerCase() === brandNameLower
+//         || list.name.toLowerCase().includes(brandNameLower)
+//         || brandNameLower.includes(list.name.toLowerCase())
+//       );
+//
+//       if (matchedList) {
+//         return matchedList.id;
+//       }
+//
+//       allLists.push(...data.result);
+//
+//       if (!data._metadata?.next) {
+//         break;
+//       }
+//
+//       const nextUrl = new URL(data._metadata.next);
+//       pageToken = nextUrl.searchParams.get('page_token');
+//
+//       if (!pageToken) {
+//         break;
+//       }
+//     }
+//
+//     if (allLists.length === 1) {
+//       return allLists[0].id;
+//     }
+//
+//     if (allLists.length > 0) {
+//       console.error(`SendGrid: No list matched brand "${brandName}". Available: ${allLists.map(l => l.name).join(', ')}`);
+//     }
+//   } catch (e) {
+//     console.error('SendGrid list lookup error:', e);
+//   }
+//
+//   return null;
+// }
+
 // --- Single Sends (Campaigns) ---
 
 /**
@@ -354,7 +387,7 @@ async function cancelSingleSend(singleSendId) {
       method: 'delete',
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     return { success: true };
@@ -375,7 +408,7 @@ async function getSingleSend(singleSendId) {
     const data = await fetch(`${BASE_URL}/marketing/singlesends/${singleSendId}`, {
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     return data.id ? data : null;
@@ -400,7 +433,7 @@ async function listSingleSends(options) {
     const data = await fetch(url, {
       response: 'json',
       headers: headers(),
-      timeout: 10000,
+      timeout: SENDGRID_TIMEOUT_MS,
     });
 
     return data.result || [];
@@ -437,7 +470,7 @@ async function addContact({ email, firstName, lastName, company, customFields })
     }
   }
 
-  const listId = await getListId();
+  const listId = getListId();
   const result = await upsertContacts({
     contacts: [contact],
     listIds: listId ? [listId] : [],
@@ -508,7 +541,7 @@ async function getSegmentContacts(segmentId, maxWaitMs = 60000) {
       const statusData = await fetch(`${BASE_URL}/marketing/contacts/exports/${exportData.id}`, {
         response: 'json',
         headers: headers(),
-        timeout: 10000,
+        timeout: SENDGRID_TIMEOUT_MS,
       });
 
       console.log(`SendGrid getSegmentContacts: Poll #${pollCount} — ${statusData.status} (${Date.now() - startTime}ms)`);

package/templates/backend-manager-config.json

@@ -138,10 +138,11 @@
   marketing: {
     sendgrid: {
       enabled: true,
+      listId: '',  // SendGrid Marketing list UUID. Populated by OMEGA's sendgrid/ensure/list.js. Skips runtime list-discovery API call.
     },
     beehiiv: {
       enabled: false,
-      // publicationId: 'pub_xxxxx',  // Set to skip fuzzy-match API call
+      publicationId: '',  // Beehiiv publication ID (e.g., 'pub_xxxxx'). Populated by OMEGA's beehiiv/ensure/publication.js. Required for marketing sync to Beehiiv.
       // Content pipeline. Lives under the provider that publishes the result —
       // Beehiiv for newsletters, eventually SendGrid for promo blasts. The
       // shape is the same regardless of provider: sources, tone, template,