backend-manager 5.8.3 → 5.9.1

package/src/manager/libraries/email/data/disposable-domains.json

@@ -260,6 +260,7 @@
   "675hosting.com",
   "675hosting.net",
   "675hosting.org",
+  "67899vip.com",
   "681mail.com",
   "689345.xyz",
   "6974.click",
@@ -879,6 +880,8 @@
   "badopsec.lol",
   "badpotato.tk",
   "bahlill.cfd",
+  "baicai1145.online",
+  "baicai1145.shop",
   "baileybridge.org",
   "baiyucraft.sbs",
   "baizoto.shop",
@@ -1200,6 +1203,7 @@
   "cantozil.com",
   "canvect.com",
   "caowo.online",
+  "capct.id",
   "capcud.shop",
   "capcut.digital",
   "capcutku.io",
@@ -1667,6 +1671,7 @@
   "devsonline.studio",
   "devthien.io.vn",
   "deyom.com",
+  "dfghjertyh.dynv6.net",
   "dfyuih.eu.cc",
   "dghdfjfd.eu.cc",
   "dh372.top",
@@ -2656,6 +2661,7 @@
   "gghjnvgfyjhk.top",
   "ggmal.ml",
   "ggrainn.com",
+  "ggvendas.store",
   "ggvk.ru",
   "ggvk.store",
   "ggxhunter.com",
@@ -3150,6 +3156,7 @@
   "http.nyc.mn",
   "httpsgreenwichmeantime.in",
   "huage.me",
+  "huangao.xyz",
   "huangniu8.com",
   "hubglee.com",
   "hubhost.store",
@@ -3512,6 +3519,7 @@
   "joseihorumon.info",
   "josse.ltd",
   "jourrapide.com",
+  "joywavepoint.com",
   "jp-ml.com",
   "jpco.org",
   "jrhmail.com",
@@ -3605,6 +3613,7 @@
   "kenfern.com",
   "kenhbanme.com",
   "kenhphim.net",
+  "kenvanharen.com",
   "kerotu.com",
   "ket-qua.org",
   "ketua.id",
@@ -4111,6 +4120,7 @@
   "mailjunk.gq",
   "mailjunk.ml",
   "mailjunk.tk",
+  "mailku.co",
   "mailllshop.website",
   "maillog.uk",
   "mailmagnet.co",
@@ -4192,6 +4202,7 @@
   "mailtv.net",
   "mailtv.tv",
   "mailuniverse.co.uk",
+  "mailvip.net",
   "mailw.site",
   "mailyan.my.id",
   "mailzi.ru",
@@ -4570,6 +4581,7 @@
   "muathegame.com",
   "muchomail.com",
   "mucincanon.com",
+  "mudahmaxwin.com",
   "muehlacker.tk",
   "muell.icu",
   "muell.io",
@@ -4686,6 +4698,7 @@
   "narsub.shop",
   "nashapoultryshop.com",
   "naslazhdai.ru",
+  "natachai.me",
   "natange.org.uk",
   "nationalgardeningclub.com",
   "natuanal.com",
@@ -4860,6 +4873,7 @@
   "notrnailinator.com",
   "notsharingmy.info",
   "novaemail.com",
+  "novaprime.vip",
   "novasub.click",
   "novyrix.cfd",
   "now.im",
@@ -4896,6 +4910,7 @@
   "nwrlr.org",
   "nwytg.com",
   "nwytg.net",
+  "nxdbxd.top",
   "ny7.me",
   "nyasan.com",
   "nyfhk.com",
@@ -5599,6 +5614,7 @@
   "rvneous.com",
   "rwstatus.com",
   "rwxsxnomo.top",
+  "rxinai.com",
   "rxs9.cn",
   "rygel.infos.st",
   "ryteto.me",
@@ -5865,6 +5881,7 @@
   "smarttalent.pw",
   "smarttickethub.com",
   "smashmail.de",
+  "smell.sbs",
   "smellfear.com",
   "smellrear.com",
   "smellypotato.tk",
@@ -7351,6 +7368,17 @@
   "yours.tools",
   "yourspamgoesto.space",
   "yourtube.ml",
+  "youtube-com-watch-jtpdc8khnpi.accesscam.org",
+  "youtube-com-watch-jtpdc8khnpi.bond",
+  "youtube-com-watch-jtpdc8khnpi.cyou",
+  "youtube-com-watch-jtpdc8khnpi.dedyn.io",
+  "youtube-com-watch-jtpdc8khnpi.freeddns.org",
+  "youtube-com-watch-jtpdc8khnpi.gleeze.com",
+  "youtube-com-watch-jtpdc8khnpi.loseyourip.com",
+  "youtube-com-watch-jtpdc8khnpi.mywire.org",
+  "youtube-com-watch-jtpdc8khnpi.ooguy.com",
+  "youtube-com-watch-jtpdc8khnpi.theworkpc.com",
+  "youtube-com-watch-jtpdc8khnpi.webredirect.org",
   "youxiang.dev",
   "yroid.com",
   "yrppwsekv.top",

package/src/manager/libraries/email/generators/lib/structure.js

@@ -211,6 +211,8 @@ async function generateStructure({ sources, brand, newsletterConfig, ai, assista
 function buildClassicSystemPrompt(brand, config) {
   const tone = config?.tone || 'professional';
   const instructions = config?.instructions || '';
+  const links = config?.links || [];
+  const keywords = config?.keywords || [];
   const taglineLine    = brand?.tagline     ? `\nTagline: ${brand.tagline}`            : '';
   const descriptionLine = brand?.description ? `\nDescription: ${brand.description}` : '';
 
@@ -218,6 +220,8 @@ function buildClassicSystemPrompt(brand, config) {
     `You are a newsletter writer for ${brand?.name || 'a tech company'}.${taglineLine}${descriptionLine}`,
     instructions ? `\nBrand instructions:\n${instructions}` : '',
     `\nTone: ${tone}`,
+    keywords.length ? `\nKeywords to weave in naturally: ${keywords.join(', ')}` : '',
+    links.length ? `\nBrand links to incorporate where relevant: ${links.join(', ')}` : '',
     '',
     'You will be given a set of "source articles" — these are background research, NOT publications you are writing for or about.',
     'Treat them as raw information. Synthesize the IDEAS into original content written as if you are the original author.',
@@ -246,7 +250,7 @@ function buildClassicSystemPrompt(brand, config) {
     '- 3-5 sections — each is ONE topic, rewritten in your voice as original content',
     '- Each section: title (compelling, scannable), body (80-150 words, markdown OK)',
     '- Each section: image_prompt — one-sentence visual description for an illustrator. Be specific about subject/style.',
-    '- Do NOT include CTAs, "read more" links, or any URLs in section bodies. The newsletter is a self-contained read — never invent links or send readers off-property.',
+    '- Do NOT invent URLs or "read more" links. The newsletter is a self-contained read. The ONLY URLs allowed in section bodies are the brand links listed above (if any) — weave them in naturally where relevant, do not force them.',
     `- Signoff: a SHORT human sign-off, formatted as two lines with \\n between them. First line is a closing phrase like "Best,", "Cheers,", "Until next week,", or "Stay sharp,". Second line is the team name like "The ${brand?.name || 'Team'} Team". Example: "Best,\\nThe ${brand?.name || 'Team'} Team". Do NOT write a summary, tagline, motto, or thematic conclusion sentence — this is the literal way you sign off the email, like the end of a letter.`,
     '- citations: array of { note, source } for any hard data referenced. Empty array if none.',
     '',

package/src/manager/libraries/email/generators/newsletter.js

@@ -7,7 +7,7 @@
  *
  * Pipeline:
  *   1. Read newsletter categories from Manager.config.marketing.newsletter.content.categories
- *   2. Fetch ready sources from parent server (atomic claim via claimFor=brandId)
+ *   2. Fetch ready sources from parent server (no claiming — child tracks locally)
  *   3. structure.js → AI authors subject, preheader, intro, sections, signoff
  *   4. image-illustrator.js → AI generates one flat-vector PNG per section via
  *      gpt-image-2 (default). Set content.method.image = 'svg' to use the legacy
@@ -21,6 +21,7 @@
  * the markdown pipeline (content) if needed.
  */
 const fetch = require('wonderful-fetch');
+const powertools = require('node-powertools');
 
 const { filterSources } = require('./lib/filter.js');
 const { generateStructure } = require('./lib/structure.js');
@@ -41,6 +42,7 @@ const { renderMarkdown } = require('./lib/markdown-renderer.js');
 const { uploadAssets, RAW_BASE } = require('./lib/image-host.js');
 const { buildPublicConfig } = require('../../../routes/brand/get.js');
 const { writeArticle, publishArticle } = require('../../../libraries/content/ghostii.js');
+const { trackContentSource, contentSourceHash } = require('../../../events/cron/daily/blog-auto-publisher.js');
 
 /**
  * Generate newsletter content from parent server sources.
@@ -69,38 +71,52 @@ const { writeArticle, publishArticle } = require('../../../libraries/content/gho
  */
 async function generate(Manager, assistant, settings, opts = {}) {
   const newsletterRoleConfig = Manager.config?.marketing?.newsletter;
-  const config = newsletterRoleConfig?.content;
+  const rawContent = newsletterRoleConfig?.content;
 
   if (!newsletterRoleConfig?.enabled) {
     assistant.log('Newsletter generator: newsletter disabled in config');
     return null;
   }
 
-  if (!config) {
+  if (!rawContent) {
     assistant.log('Newsletter generator: no marketing.newsletter.content config block');
     return null;
   }
 
-  // Either use pre-fetched sources (iteration test) or fetch from parent
+  // Content can be an array or single object (matches blog config shape)
+  const contentArray = powertools.arrayify(rawContent);
+  const config = contentArray[0];
+
+  if (!config) {
+    assistant.log('Newsletter generator: empty content array');
+    return null;
+  }
+
+  // Default sources to ['$parent'] if not specified
+  const contentSources = config.sources || ['$parent'];
+
+  // Either use pre-fetched sources (iteration test) or resolve from config
   let sources = opts.sources;
 
   if (!sources) {
-    const categories = config.categories || [];
+    // For now, newsletter resolves $parent sources via the parent server
+    if (contentSources.includes('$parent')) {
+      const categories = config.categories || [];
 
-    if (!categories.length) {
-      assistant.log('Newsletter generator: no categories configured');
-      return null;
-    }
+      if (!categories.length) {
+        assistant.log('Newsletter generator: no categories configured');
+        return null;
+      }
 
-    const parentUrl = Manager.getParentApiUrl();
+      const parentUrl = Manager.getParentApiUrl();
 
-    if (!parentUrl) {
-      assistant.log('Newsletter generator: no parent URL configured');
-      return null;
-    }
+      if (!parentUrl) {
+        assistant.log('Newsletter generator: no parent URL configured');
+        return null;
+      }
 
-    const brandId = Manager.config?.brand?.id;
-    sources = await fetchSources(parentUrl, categories, brandId, assistant);
+      sources = await fetchSources(parentUrl, categories, assistant);
+    }
   }
 
   if (!sources?.length) {
@@ -382,12 +398,20 @@ async function generate(Manager, assistant, settings, opts = {}) {
     });
   }
 
-  // 4. Mark sources as used on parent server (unless caller opted out)
-  if (!opts.skipClaim) {
-    const parentUrl = Manager.getParentApiUrl();
-
-    if (parentUrl) {
-      await claimSources(parentUrl, sources, brand?.id, assistant);
+  // 4. Track all sources in the unified content-sources collection
+  const admin = Manager.libraries?.admin;
+  if (admin) {
+    for (const source of sources) {
+      await trackContentSource(admin, {
+        url: source.url || source.id,
+        origin: '$parent',
+        itemId: source.id,
+        itemTitle: source.title || '',
+        usedBy: 'newsletter',
+        brandId: brand?.id || '',
+      }).catch((e) => {
+        assistant.error(`Newsletter generator: Error tracking content source (non-fatal): ${e.message}`);
+      });
     }
   }
 
@@ -675,7 +699,7 @@ function aggregateTotals(filterMeta, structureMeta, images) {
 /**
  * Fetch ready newsletter sources from the parent server.
  */
-async function fetchSources(parentUrl, categories, brandId, assistant) {
+async function fetchSources(parentUrl, categories, assistant) {
   const allSources = [];
 
   for (const category of categories) {
@@ -687,7 +711,6 @@ async function fetchSources(parentUrl, categories, brandId, assistant) {
         query: {
           category,
           limit: 3,
-          claimFor: brandId,
           backendManagerKey: process.env.BACKEND_MANAGER_KEY,
         },
       });
@@ -703,27 +726,6 @@ async function fetchSources(parentUrl, categories, brandId, assistant) {
   return allSources;
 }
 
-/**
- * Mark sources as used on the parent server.
- */
-async function claimSources(parentUrl, sources, brandId, assistant) {
-  for (const source of sources) {
-    try {
-      await fetch(`${parentUrl}/newsletter-sources`, {
-        method: 'put',
-        response: 'json',
-        timeout: 60000,
-        body: {
-          id: source.id,
-          usedBy: brandId || 'unknown',
-          backendManagerKey: process.env.BACKEND_MANAGER_KEY,
-        },
-      });
-    } catch (e) {
-      assistant.error(`Newsletter generator: Failed to claim source ${source.id}: ${e.message}`);
-    }
-  }
-}
 
 /**
  * Generate a 20-character Firebase push ID (RTDB-style).
@@ -776,4 +778,4 @@ function generatePushId() {
   return id;
 }
 
-module.exports = { generate, fetchSources, claimSources, generatePushId };
+module.exports = { generate, fetchSources, generatePushId };

package/src/manager/routes/brand/get.js

@@ -10,7 +10,7 @@ module.exports = async ({ assistant, Manager }) => {
 
 /**
  * Build a public-safe config object from Manager.config
- * Excludes sensitive fields: sentry, analytics, ghostii, etc.
+ * Excludes sensitive fields: sentry, analytics, blog, etc.
  */
 function buildPublicConfig(config) {
   return {

package/templates/backend-manager-config.json

@@ -146,41 +146,29 @@
       enabled: false,
       platform: 'beehiiv',
       publicationId: '',  // Beehiiv publication ID (e.g., 'pub_xxxxx'). Populated by OMEGA's beehiiv/ensure/publication.js. Required for marketing sync to Beehiiv.
-      content: {
-        categories: [],  // e.g., ['social-media', 'marketing'] — content categories to pull from parent server
-        // AI-customization fields below are all optional with sensible defaults
-        instructions: '',  // free-form text passed to the AI ("focus on X", "avoid Y", brand voice notes)
-        tone: 'professional',  // 'professional', 'casual', 'actionable', 'witty', etc. — passed to AI prompt
-        template: 'clean',   // 'clean' | 'editorial' | 'field-report' — layout template (each owns its own content shape and aesthetic)
-        article: {
-          enabled: false,  // when true, the newsletter's lead section is expanded into a full blog post via Ghostii, published to the website repo, and linked from the newsletter with a "Read the full article" CTA
-          author: null,  // author slug for the linked article (Ghostii/admin-post picks a default if unset)
-        },
-        theme: {
-          primaryColor: '#5B5BFF',   // accent color: buttons, links, brand text
-          secondaryColor: '#1E1E2A', // body text color
-          accentColor: '#F6F7FB',    // page background outside the 600px body
-          font: 'Inter, system-ui, sans-serif',
-        },
-        // AI provider defaults are hard-coded in the library (openai for structure,
-        // anthropic for SVG — each chosen for what each model does best). Override
-        // per-run only via env: NEWSLETTER_PROVIDER_STRUCTURE / NEWSLETTER_PROVIDER_SVG.
-        // Brand-owned sponsorship/promo slots — rendered as "sponsored" blocks inline
-        // with the newsletter. Use for promoting your own products, affiliate offers,
-        // or paid placements. Each entry can be positioned at 'top', 'middle', or 'end'.
-        // Per-campaign overrides via settings.sponsorships on a marketing-campaigns doc.
-        sponsorships: [
-          // {
-          //   label:       'Sponsored',                 // optional eyebrow label (defaults to "Sponsored")
-          //   headline:    'Grow your audience faster', // bold headline
-          //   body:        'Short pitch text…',         // 1-2 sentence body
-          //   url:         'https://somiibo.com/promo', // click destination
-          //   image:       'https://...png',            // optional image URL
-          //   ctaLabel:    'Try Somiibo',               // button text (default: "Learn more")
-          //   position:    'middle',                    // 'top' | 'middle' | 'end' (default: 'middle')
-          // },
-        ],
-      },
+      // Content: single object or array (matches blog config shape). Generator uses the first entry.
+      content: [
+        {
+          sources: ['$parent'],    // '$parent' | '$feed:https://...' | '$brand'
+          categories: [],          // e.g., ['social-media', 'marketing'] — topic constraints + AI categorization
+          links: [],               // links to inject into generated content
+          instructions: '',        // free-form text passed to the AI
+          tone: 'professional',    // 'professional' | 'casual' | 'actionable' | 'witty' | etc.
+          keywords: [],            // SEO keywords to weave in
+          template: 'clean',       // 'clean' | 'editorial' | 'field-report' — layout template
+          article: {
+            enabled: false,        // expand lead section into a full blog post via Ghostii
+            author: null,
+          },
+          theme: {
+            primaryColor: '#5B5BFF',
+            secondaryColor: '#1E1E2A',
+            accentColor: '#F6F7FB',
+            font: 'Inter, system-ui, sans-serif',
+          },
+          sponsorships: [],
+        },
+      ],
     },
     prune: {
       enabled: true,
@@ -195,48 +183,56 @@
     appId: '1:123:web:456',
     measurementId: 'G-0123456789',
   },
-  // Standalone Ghostii article publisher (daily cron). OPT-IN: disabled by
-  // default (articles: 0). Set articles >= 1 to auto-publish independent blog
-  // posts. For newsletter-linked articles, use marketing.newsletter.content.article.enabled instead.
+  // Standalone blog auto-publisher (daily cron). OPT-IN: disabled by default.
+  // Set enabled: true and quantity >= 1 to auto-publish independent blog posts.
+  // For newsletter-linked articles, use marketing.newsletter.content.article.enabled instead.
+  //
+  // `platform` selects the article-generation provider (only 'ghostii' for now).
   //
   // Source types:
-  //   '$app'                            — generic brand-topic generation
+  //   '$brand'                          — generic brand-topic generation
   //   '$feed:https://example.com/feed'  — RSS/Atom/JSON feed: pick one unprocessed article per run
+  //   '$parent'                         — fetch sources from parent server's source pool (without claiming)
   //   'https://example.com/page'        — fetch URL content as prompt seed
   //   'Write about topic X'             — text: use directly as prompt seed
   //
-  // Feed items are tracked in Firestore (ghostii-feed-items) so the same article
-  // is never reprocessed. When a feed is unreachable or exhausted, falls back to $app.
-  ghostii: [
-    {
-      articles: 0,
-      sources: [
-        '$app',
-        // '$feed:https://techcrunch.com/feed/',
-        // '$feed:https://feeds.arstechnica.com/arstechnica/technology-lab',
-      ],
-      links: [
-        // ...
-      ],
-      prompt: '',
-      chance: 1.0,
-      author: null,
-      postPath: 'ghostii',
-      // Per-entry Ghostii API overrides (all optional, fall back to framework defaults)
-      overrides: {
-        // keywords: ['AI', 'technology'],
-        // length: 'long',              // short | medium | long | comprehensive
-        // research: true,              // web search for real external links
-        // insertImages: true,          // section images from Unsplash
-        // headerImageUrl: 'unsplash',  // disabled | unsplash | generate
-        // maxLinks: 6,
-        // sectionQuantity: 5,
-        // feedUrl: '',                 // brand blog feed for title dedup
+  // All feed/parent sources are tracked in Firestore (`content-sources`) so the same
+  // article is never reprocessed. When a feed is unreachable or exhausted, falls back to $brand.
+  blog: {
+    enabled: false,
+    platform: 'ghostii',
+    content: [
+      {
+        sources: [             // '$brand' | '$feed:https://...' | '$parent'
+          '$brand',
+          // '$feed:https://techcrunch.com/feed/',
+          // '$parent',
+        ],
+        categories: [],        // topic constraints + output tags
+        links: [],             // links to inject into generated content
+        instructions: '',      // free-form text passed to the AI
+        tone: 'professional',  // 'professional' | 'casual' | 'actionable' | 'witty' | etc.
+        keywords: [],          // SEO keywords to weave in
+        quantity: 0,           // number of articles to generate per run
+        chance: 1.0,
+        author: null,
+        postPath: 'ghostii',  // defaults to platform name; sub-folder under _posts/{year}/
+        // Per-entry Ghostii API overrides (all optional, fall back to framework defaults)
+        overrides: {
+          // keywords: ['AI', 'technology'],
+          // length: 'long',              // short | medium | long | comprehensive
+          // research: true,              // web search for real external links
+          // insertImages: true,          // section images from Unsplash
+          // headerImageUrl: 'unsplash',  // disabled | unsplash | generate
+          // maxLinks: 6,
+          // sectionQuantity: 5,
+          // feedUrl: '',                 // brand blog feed for title dedup
+        },
+        // brand: 'other-app-id',                // Optional: target a different brand
+        // brandUrl: 'https://api.otherapp.com',  // Required if brand is set
       },
-      // brand: 'other-app-id',                // Optional: target a different brand
-      // brandUrl: 'https://api.otherapp.com',  // Required if brand is set
-    }
-  ],
+    ],
+  },
   reviews: {
     enabled: true,
     sites: [