backend-manager 5.2.15 → 5.2.17

package/test/marketing/newsletter-generate.js

@@ -44,8 +44,16 @@
  *   NEWSLETTER_PEEK=1                  Fetch + list ready sources, do not claim, exit.
  *   NEWSLETTER_SOURCE_ID=<id>          Generate from one specific source WITHOUT claiming it.
  *   NEWSLETTER_LIMIT=10                Sources per category for PEEK (default 10).
+ *   NEWSLETTER_CLAIM=1               CLAIM the fetched sources (claimFor=brandId), consuming them so the
+ *                                       real newsletter won't reuse them. OFF by default — the test fetches
+ *                                       without claiming so runs are repeatable and non-destructive.
  *   NEWSLETTER_RELEASE=1               Reset locally-tracked claimed sources back to 'ready'.
  *   NEWSLETTER_NO_IMAGES=1             Skip SVG/PNG generation (fast iteration on copy only).
+ *   NEWSLETTER_CREATE_ARTICLE=1       PUBLISH the linked blog article to the website repo (Ghostii → admin/post → GitHub).
+ *                                       The article is always GENERATED when the brand's
+ *                                       marketing.beehiiv.content.article.enabled is on (exercises the Ghostii write +
+ *                                       URL/CTA path); this flag only controls whether it's actually committed.
+ *                                       OFF by default — a newsletter test run never commits a real post.
  *   NEWSLETTER_PROVIDER_STRUCTURE=X    Override structure provider (openai|anthropic).
  *   NEWSLETTER_PROVIDER_SVG=X          Override SVG provider (openai|anthropic).
  *   NEWSLETTER_CAMPAIGN_ID=<id>        Override the auto-generated campaign ID (folder name in newsletter-assets).
@@ -345,13 +353,20 @@ module.exports = {
       return;
     }
 
-    // --- Fetch sources (real claim) ---
+    // --- Fetch sources ---
+    // By DEFAULT the test does NOT claim sources — it fetches them without
+    // claimFor, so they stay 'ready' and available for the real newsletter.
+    // This keeps test runs repeatable and non-destructive (a test should never
+    // silently consume production resources). Set NEWSLETTER_CLAIM=1 to exercise
+    // the real claim/consume path (then use NEWSLETTER_RELEASE=1 to put them back).
+    const claim = !!env.NEWSLETTER_CLAIM;
     const sources = await fetchSourcesForRun({
       parentUrl,
       newsletterConfig,
       brandId: config.brand?.id,
       sourceId: env.NEWSLETTER_SOURCE_ID,
       key: env.BACKEND_MANAGER_KEY,
+      claim,
     });
 
     // Environmental precondition: the parent server must have ready sources in
@@ -361,8 +376,10 @@ module.exports = {
       return skip('No ready newsletter sources available on parent server (environmental)');
     }
 
-    // Track claimed IDs for later --release-all
-    appendClaimed(claimedFile, sources.map((s) => s.id));
+    // Track claimed IDs for later --release-all (only when we actually claimed)
+    if (claim && !env.NEWSLETTER_SOURCE_ID) {
+      appendClaimed(claimedFile, sources.map((s) => s.id));
+    }
 
     // Force `beehiiv.enabled: true` and inject the per-run newsletter config
     // overrides onto Manager.config. The iteration test IS the explicit trigger
@@ -382,12 +399,18 @@ module.exports = {
     // --- Run the production generator with the local-persist image hook ---
     const generator = require('../../src/manager/libraries/email/generators/newsletter.js');
 
-    // EXTENDED mode is a MIRROR of the production cron — no escape hatches.
+    // EXTENDED mode mirrors the production cron's newsletter side effects:
     // GH upload always happens (PNGs + newsletter.html), Beehiiv draft upload
     // always happens (governed inside newsletter.js by beehiiv.enabled, which
     // we force true above). If you don't want the side effects, run fixture
     // mode instead.
     //
+    // The ONE deliberate exception is PUBLISHING the linked blog article: the
+    // article is still generated (so the test exercises the Ghostii write + CTA
+    // path), but it's NOT committed to the website repo unless you opt in with
+    // NEWSLETTER_CREATE_ARTICLE=1 (publishArticle below). Committing a real post
+    // is out of scope for a routine newsletter test.
+    //
     // persistImage is a side-effect callback that writes PNG+SVG to runDir for
     // local preview / debug. Its return value is ignored when imageHost: 'github'
     // because the generator uses the uploaded CDN URLs in the rendered HTML.
@@ -416,6 +439,11 @@ module.exports = {
         sources,
         skipClaim: true, // We manage the claim/release lifecycle ourselves
         skipImages: !!env.NEWSLETTER_NO_IMAGES,
+        // The article is GENERATED whenever the brand's config.article.enabled is on
+        // (exercises the Ghostii write + URL/CTA path), but only PUBLISHED to the
+        // website repo when you opt in with NEWSLETTER_CREATE_ARTICLE=1. Default
+        // test run generates but does not commit a real post.
+        publishArticle: !!env.NEWSLETTER_CREATE_ARTICLE,
         // Local disk persistence runs unconditionally (for preview/debug)
         persistImage,
         // EXTENDED always uploads to GitHub — mirrors production cron exactly
@@ -444,10 +472,37 @@ module.exports = {
       assets: result.assets || null,
     }, null, 2));
 
+    // Linked blog article (when content.article.enabled). Save the full Ghostii
+    // output + the computed URL so you can review what would be published — both
+    // the raw JSON and a readable markdown view of the article body.
+    if (result.article?.article) {
+      const a = result.article.article;
+      jetpack.write(path.join(runDir, 'article.json'), JSON.stringify(result.article, null, 2));
+      jetpack.write(path.join(runDir, 'article.md'), [
+        `# ${a.title || ''}`,
+        '',
+        `> ${a.description || ''}`,
+        '',
+        `**URL:** ${result.article.url || '(none)'}`,
+        `**Published:** ${result.article.published ? 'yes' : 'no (generate-only)'}`,
+        a.headerImageUrl ? `**Header image:** ${a.headerImageUrl}` : '',
+        a.categories?.length ? `**Categories:** ${a.categories.join(', ')}` : '',
+        a.keywords?.length ? `**Keywords:** ${a.keywords.join(', ')}` : '',
+        '',
+        '---',
+        '',
+        a.body || '',
+      ].filter(line => line !== undefined).join('\n'));
+    }
+
     console.log(`\nNewsletter preview written: ${previewPath}`);
     console.log(`Subject:   ${result.subject}`);
     console.log(`Preheader: ${result.preheader}`);
     console.log(`Sections:  ${result.structure.sections.length}`);
+    if (result.article?.article) {
+      console.log(`Article:   "${result.article.article.title}" → ${result.article.url} (${result.article.published ? 'published' : 'generate-only'})`);
+      console.log(`           ${path.join(runDir, 'article.md')}`);
+    }
     if (result.meta?.totals) {
       const t = result.meta.totals;
       console.log(`\nRun summary:`);
@@ -606,13 +661,14 @@ async function peekSources({ parentUrl, categories, limit, key }) {
 /**
  * Fetch sources for an actual generation run. Either:
  *   - A specific source by id — preview only, NO claim (iterate repeatedly on the same source)
- *   - Or N per category (claims them atomically for the brand)
+ *   - Or N per category. Fetches WITHOUT claiming by default (claim=false), so runs are
+ *     repeatable; pass claim=true (NEWSLETTER_CLAIM=1) to atomically claim/consume them.
  *
  * When NEWSLETTER_SOURCE_ID is set, we look the source up in any status
  * (ready, claimed, used) so you can keep iterating on it across runs without
  * the parent server's claim mechanism marking it consumed.
  */
-async function fetchSourcesForRun({ parentUrl, newsletterConfig, brandId, sourceId, key }) {
+async function fetchSourcesForRun({ parentUrl, newsletterConfig, brandId, sourceId, key, claim = false }) {
   if (sourceId) {
     // Peek across ALL ready sources (no claim). Search broadly first, then
     // fall back to any-status if needed. We never call claimFor with sourceId
@@ -657,11 +713,18 @@ async function fetchSourcesForRun({ parentUrl, newsletterConfig, brandId, source
 
   const all = [];
   for (const category of categories) {
+    // claimFor is what tells the parent server to mark sources consumed. Omit it
+    // (claim=false) to fetch the same sources without claiming them.
+    const query = { category, limit: 3, backendManagerKey: key };
+    if (claim) {
+      query.claimFor = brandId;
+    }
+
     const data = await fetch(`${parentUrl}/newsletter-sources`, {
       method: 'get',
       response: 'json',
       timeout: 15000,
-      query: { category, limit: 3, claimFor: brandId, backendManagerKey: key },
+      query,
     });
     all.push(...(data.sources || []));
   }

package/test/routes/user/signup.js

@@ -347,6 +347,174 @@ module.exports = {
       },
     },
 
+    // --- Consent downgrade-protection tests ---
+    // Guards against data loss when a LEGACY account (signed up before the flags.signupProcessed
+    // flow existed, so flag never set) re-fires /user/signup on page load. Its localStorage
+    // consent is long gone, so the payload is empty — without the guard, buildConsentRecord
+    // would compute 'revoked' and the {merge:true} write would wipe the consent the user
+    // actually granted months ago. The guard preserves any existing 'granted' status.
+    {
+      name: 'consent-empty-payload-preserves-existing-grant',
+      async run({ http, firestore, assert, accounts }) {
+        const uid = accounts['consent-preserve'].uid;
+
+        // Seed the doc as an established account whose consent is already granted (as a real
+        // legacy signup would be after the OMEGA migration backfilled consent). flags is left
+        // at the schema default (signupProcessed: false) to mimic the legacy state exactly.
+        // merge:true — preserve the runner-provisioned auth.uid (pollForUserDoc needs it).
+        await firestore.set(`users/${uid}`, {
+          consent: {
+            legal: {
+              status: 'granted',
+              grantedAt: { timestamp: '2025-01-01T00:00:00.000Z', timestampUNIX: 1735689600, source: 'signup', ip: null, text: 'Legacy legal grant' },
+            },
+            marketing: {
+              status: 'granted',
+              grantedAt: { timestamp: '2025-01-01T00:00:00.000Z', timestampUNIX: 1735689600, source: 'signup', ip: null, text: 'Legacy marketing grant' },
+              revokedAt: { timestamp: null, timestampUNIX: null, source: null, ip: null, text: null },
+            },
+          },
+          flags: { signupProcessed: false },
+        }, { merge: true });
+
+        // Re-fire signup with NO consent payload (the legacy page-load case).
+        const signupResponse = await http.as('consent-preserve').post('user/signup', {});
+        assert.isSuccess(signupResponse, `Signup should succeed: ${JSON.stringify(signupResponse, null, 2)}`);
+
+        const userDoc = await firestore.get(`users/${uid}`);
+
+        // CRITICAL: the prior grants must survive — NOT be downgraded to revoked.
+        assert.equal(userDoc?.consent?.legal?.status, 'granted', 'legal.status must stay granted (not downgraded by empty payload)');
+        assert.equal(userDoc?.consent?.legal?.grantedAt?.text, 'Legacy legal grant', 'legal grantedAt must be the preserved original, not wiped');
+        assert.equal(userDoc?.consent?.legal?.grantedAt?.timestampUNIX, 1735689600, 'legal grantedAt timestamp must be preserved');
+
+        assert.equal(userDoc?.consent?.marketing?.status, 'granted', 'marketing.status must stay granted (not downgraded by empty payload)');
+        assert.equal(userDoc?.consent?.marketing?.grantedAt?.text, 'Legacy marketing grant', 'marketing grantedAt must be the preserved original');
+        // No spurious revokedAt should have been stamped over the preserved grant.
+        assert.equal(userDoc?.consent?.marketing?.revokedAt?.timestamp, null, 'marketing revokedAt must stay null (no decline was recorded)');
+
+        // signupProcessed should now be flipped true by this run.
+        assert.equal(userDoc?.flags?.signupProcessed, true, 'signupProcessed should be set true after the run');
+      },
+    },
+    {
+      name: 'consent-explicit-decline-does-not-downgrade-existing-grant',
+      async run({ http, firestore, assert, accounts }) {
+        const uid = accounts['consent-preserve'].uid;
+
+        // Re-seed: granted marketing + UNSET signupProcessed so the route processes this call.
+        // Then send a payload that explicitly DECLINES marketing. The guard must still preserve
+        // the existing grant — only an explicit RE-GRANT may overwrite; a decline-over-grant on
+        // the signup path is treated as a non-grant and must not wipe a prior consent.
+        // merge:true — preserve the runner-provisioned auth.uid (pollForUserDoc needs it).
+        await firestore.set(`users/${uid}`, {
+          consent: {
+            marketing: {
+              status: 'granted',
+              grantedAt: { timestamp: '2025-01-01T00:00:00.000Z', timestampUNIX: 1735689600, source: 'signup', ip: null, text: 'Prior marketing grant' },
+              revokedAt: { timestamp: null, timestampUNIX: null, source: null, ip: null, text: null },
+            },
+          },
+          flags: { signupProcessed: false },
+        }, { merge: true });
+
+        const signupResponse = await http.as('consent-preserve').post('user/signup', {
+          consent: {
+            legal: { granted: true, text: 'Legal grant on re-fire' },
+            marketing: { granted: false, text: 'Declining marketing' },
+          },
+        });
+        assert.isSuccess(signupResponse, `Signup should succeed: ${JSON.stringify(signupResponse, null, 2)}`);
+
+        const userDoc = await firestore.get(`users/${uid}`);
+
+        // Legal newly granted this call.
+        assert.equal(userDoc?.consent?.legal?.status, 'granted', 'legal.status should be granted from this call');
+        // Marketing was already granted; an explicit decline must NOT downgrade it.
+        assert.equal(userDoc?.consent?.marketing?.status, 'granted', 'marketing.status must stay granted (decline cannot downgrade an existing grant on signup path)');
+        assert.equal(userDoc?.consent?.marketing?.grantedAt?.text, 'Prior marketing grant', 'marketing grant must be the preserved original');
+      },
+    },
+
+    // --- buildUserRecord layered deep-merge tests ---
+    // The signup write must: (a) fill every schema leaf so the doc is complete (no migration
+    // churn), (b) PRESERVE existing real values (api keys, subscription, roles, affiliate.code,
+    // custom non-schema fields), and (c) apply the signup data on top — without Firestore's
+    // map-replace wiping nested data. These tests seed adversarial existing state and verify.
+    {
+      name: 'merge-preserves-existing-and-fills-schema',
+      async run({ http, firestore, assert, accounts }) {
+        const uid = accounts['signup-merge'].uid;
+
+        // Seed real/custom state that the signup write must NOT clobber, plus a deliberately
+        // PARTIAL attribution (only affiliate.code) to prove leaves get filled, not replaced-away.
+        await firestore.set(`users/${uid}`, {
+          api: { clientId: 'REAL-CLIENT-ID', privateKey: 'REAL-PRIVATE-KEY' },
+          affiliate: { code: 'REALAFFILIATECODE', referrals: [{ uid: 'someone' }] },
+          subscription: { product: { id: 'pro', name: 'Pro' }, status: 'active' },
+          roles: { admin: true, betaTester: false, developer: false },
+          attribution: { affiliate: { code: 'PARTIALONLY' } },
+          myCustomIntegration: { slackWebhook: 'https://hooks.slack.com/services/XXX' },
+        }, { merge: true });
+
+        const signupResponse = await http.as('signup-merge').post('user/signup', {
+          consent: { legal: { granted: true, text: 'I agree.' }, marketing: { granted: true, text: 'Updates please.' } },
+          attribution: { utm: { tags: { utm_source: 'newsletter' } } },
+        });
+        assert.isSuccess(signupResponse, `Signup should succeed: ${JSON.stringify(signupResponse, null, 2)}`);
+
+        const doc = await firestore.get(`users/${uid}`);
+
+        // (b) Existing real values must survive untouched — NOT regenerated/reset by the schema layer.
+        assert.equal(doc?.api?.clientId, 'REAL-CLIENT-ID', 'api.clientId must be preserved (not regenerated)');
+        assert.equal(doc?.api?.privateKey, 'REAL-PRIVATE-KEY', 'api.privateKey must be preserved (not regenerated)');
+        assert.equal(doc?.affiliate?.code, 'REALAFFILIATECODE', 'affiliate.code must be preserved (not regenerated)');
+        assert.equal(doc?.subscription?.product?.id, 'pro', 'subscription must be preserved (not reset to basic)');
+        assert.equal(doc?.roles?.admin, true, 'roles.admin must be preserved (not reset to false)');
+
+        // (b) Custom non-schema field must survive the merge.
+        assert.equal(doc?.myCustomIntegration?.slackWebhook, 'https://hooks.slack.com/services/XXX', 'custom non-schema field must survive');
+
+        // (a) Every attribution leaf must be present (the bug: partial write flattened the map).
+        assert.hasProperty(doc, 'attribution.affiliate.code', 'attribution.affiliate.code must exist');
+        assert.hasProperty(doc, 'attribution.affiliate.url', 'attribution.affiliate.url must exist (filled)');
+        assert.hasProperty(doc, 'attribution.affiliate.page', 'attribution.affiliate.page must exist (filled)');
+        assert.hasProperty(doc, 'attribution.affiliate.timestamp', 'attribution.affiliate.timestamp must exist (filled)');
+        assert.hasProperty(doc, 'attribution.utm.url', 'attribution.utm.url must exist (filled)');
+        assert.hasProperty(doc, 'attribution.utm.page', 'attribution.utm.page must exist (filled)');
+        assert.hasProperty(doc, 'attribution.utm.timestamp', 'attribution.utm.timestamp must exist (filled)');
+        // filled leaves should be null, not undefined/missing
+        assert.equal(doc?.attribution?.affiliate?.url, null, 'unset attribution leaf should be null');
+
+        // (c) Signup data applied on top.
+        assert.equal(doc?.attribution?.affiliate?.code, 'PARTIALONLY', 'pre-existing affiliate.code preserved (signup did not send one)');
+        assert.equal(doc?.attribution?.utm?.tags?.utm_source, 'newsletter', 'signup utm tag applied');
+        assert.equal(doc?.flags?.signupProcessed, true, 'flags.signupProcessed set true');
+        assert.equal(doc?.consent?.legal?.status, 'granted', 'consent applied');
+      },
+    },
+    {
+      name: 'merge-fills-all-leaves-on-schema-complete-doc',
+      async run({ http, firestore, assert, accounts }) {
+        // Sanity: after signup, the doc must contain the full set of top-level schema sections,
+        // so a subsequent migration finds NOTHING to backfill. Reuses the signup-merge account
+        // (already processed above → re-fire is rejected, but the doc from the prior test is the
+        // artifact we assert against; this test just validates that doc's completeness).
+        const uid = accounts['signup-merge'].uid;
+        const doc = await firestore.get(`users/${uid}`);
+
+        for (const section of ['auth', 'roles', 'flags', 'affiliate', 'metadata', 'activity', 'api', 'personal', 'attribution', 'consent', 'subscription']) {
+          assert.hasProperty(doc, section, `doc must have top-level '${section}' section after signup`);
+        }
+        // Nested completeness spot-checks across the sections signup writes.
+        assert.hasProperty(doc, 'activity.geolocation.ip', 'activity.geolocation.ip must exist');
+        assert.hasProperty(doc, 'activity.client.userAgent', 'activity.client.userAgent must exist');
+        assert.hasProperty(doc, 'personal.name.first', 'personal.name.first must exist');
+        assert.hasProperty(doc, 'consent.marketing.revokedAt.source', 'consent.marketing.revokedAt.source must exist');
+        assert.hasProperty(doc, 'metadata.created.timestampUNIX', 'metadata.created.timestampUNIX must exist');
+      },
+    },
+
     // --- Auth rejection test (at end per convention) ---
     {
       name: 'unauthenticated-rejected',