backend-manager 5.0.202 → 5.1.0

package/src/manager/libraries/disposable-domains.json

@@ -1997,6 +1997,7 @@
   "gmail2.gq",
   "gmailot.com",
   "gmatch.org",
+  "gmeenramy.com",
   "gmial.com",
   "gmx1mail.top",
   "gmxmail.top",
@@ -4041,6 +4042,7 @@
   "rustydoor.com",
   "rustyload.com",
   "ruu.kr",
+  "ruutukf.com",
   "rvb.ro",
   "rwstatus.com",
   "rygel.infos.st",

package/src/manager/libraries/email/generators/lib/filter.js

@@ -0,0 +1,179 @@
+/**
+ * Brand-fit filter for newsletter sources.
+ *
+ * Given a pool of raw newsletter sources and a brand's `marketing.beehiiv.content`
+ * config, asks an AI to score each source for brand fit (0-10), then drops
+ * anything below the threshold. This prevents off-topic sources from leaking
+ * into the structure generator.
+ *
+ * Fit scoring is a single AI call (one round-trip for the whole pool), using
+ * a small/cheap model by default.
+ */
+const DEFAULT_MODELS = {
+  openai: 'gpt-5.4-mini',
+  anthropic: 'claude-opus',
+  'claude-code': 'claude-opus-4-7',
+};
+
+const DEFAULT_THRESHOLD = 6; // 0-10 scale; only sources scoring >= this make it through
+
+const FILTER_SCHEMA = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['scores'],
+  properties: {
+    scores: {
+      type: 'array',
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['id', 'fit', 'reason'],
+        properties: {
+          id:     { type: 'string' },
+          fit:    { type: 'integer', minimum: 0, maximum: 10 },
+          reason: { type: 'string' },
+        },
+      },
+    },
+  },
+};
+
+/**
+ * Score sources for brand fit and return only the ones above threshold.
+ *
+ * @param {object} args
+ * @param {Array<object>} args.sources - raw source records
+ * @param {object} args.brand - { name, description }
+ * @param {object} args.newsletterConfig
+ * @param {object} args.ai
+ * @param {object} args.assistant
+ * @param {number} [args.threshold] - override fit threshold (default 6)
+ * @returns {Promise<{kept: object[], scores: object[]}>}
+ */
+async function filterSources({ sources, brand, newsletterConfig, ai, assistant, threshold }) {
+  if (!sources?.length) {
+    return { kept: [], scores: [] };
+  }
+
+  const t = typeof threshold === 'number' ? threshold : DEFAULT_THRESHOLD;
+  const provider = newsletterConfig?.provider?.filter || 'openai';
+  const model = newsletterConfig?.model?.filter || DEFAULT_MODELS[provider];
+  const startTime = Date.now();
+
+  assistant.log(`Newsletter filter: scoring ${sources.length} sources (provider=${provider} threshold=${t})`);
+
+  const systemPrompt = buildSystemPrompt(brand, newsletterConfig);
+  const userPrompt = buildUserPrompt(sources);
+
+  let scores = [];
+  let aiResult = null;
+
+  try {
+    aiResult = await ai.request({
+      provider,
+      model,
+      messages: [
+        { role: 'system', content: systemPrompt },
+        { role: 'user',   content: userPrompt },
+      ],
+      response: 'json',
+      schema: FILTER_SCHEMA,
+      maxTokens: 8000,
+      temperature: 0.2,
+      moderate: false,
+    });
+
+    scores = aiResult.content?.scores || [];
+
+    if (!scores.length) {
+      assistant.log(`Newsletter filter: AI returned no scores. Raw content: ${JSON.stringify(aiResult.content)?.slice(0, 500)}`);
+    }
+  } catch (e) {
+    assistant.error(`Filter failed: ${e.message}. Falling back to no filtering.`);
+    return {
+      kept: sources,
+      scores: [],
+      meta: { provider, model, durationMs: Date.now() - startTime, error: e.message },
+    };
+  }
+
+  // Index scores by source id
+  const scoreById = new Map(scores.map((s) => [s.id, s]));
+
+  // Log all scores up front for visibility (sorted highest fit first)
+  const sortedScores = [...scores].sort((a, b) => b.fit - a.fit);
+  for (const s of sortedScores) {
+    assistant.log(`  fit=${s.fit} ${s.id} — ${s.reason}`);
+  }
+
+  // Keep sources scoring >= threshold; preserve original order
+  const kept = sources.filter((s) => {
+    const score = scoreById.get(s.id);
+    return score && score.fit >= t;
+  });
+
+  assistant.log(`Newsletter filter: kept ${kept.length}/${sources.length} sources (threshold=${t})`);
+
+  return {
+    kept,
+    scores,
+    meta: {
+      provider,
+      model,
+      durationMs: Date.now() - startTime,
+      threshold: t,
+      sourcesIn: sources.length,
+      sourcesKept: kept.length,
+      tokens: aiResult?.tokens || null,
+    },
+  };
+}
+
+function buildSystemPrompt(brand, config) {
+  const tone = config?.tone || 'professional';
+  const instructions = config?.instructions || '';
+  const categories = (config?.categories || []).join(', ') || 'general';
+
+  return [
+    `You evaluate which third-party newsletter content is on-brand for ${brand?.name || 'a brand'} to feature in their own newsletter.`,
+    brand?.tagline     ? `\n${brand.name} tagline: ${brand.tagline}`         : '',
+    brand?.description ? `\n${brand.name} description: ${brand.description}` : '',
+    instructions       ? `\nNewsletter focus: ${instructions}`               : '',
+    `\nContent categories this brand wants: ${categories}`,
+    `\nIntended tone: ${tone}`,
+    '',
+    'The user will give you a JSON array of source articles (forwarded newsletter excerpts).',
+    'For EACH source, judge whether the topic fits this brand\'s audience and assign a fit score 0–10:',
+    '  10 = directly about this brand\'s domain — their audience definitely cares',
+    '   7 = adjacent/related topic — plausible audience interest',
+    '   4 = tangentially relevant',
+    '   0 = off-topic — wrong audience or wrong domain',
+    '',
+    'Be honest and discerning — off-topic content should score low even if interesting.',
+    'Fit is about whether THIS BRAND\'S audience would care, not about the article\'s general quality.',
+    '',
+    'Output schema: { "scores": [{ "id": string, "fit": integer, "reason": string }] }',
+    'RULES:',
+    '  • Use the EXACT id value from each input source (e.g. "-OsTDh6dAWqUQrcq7B3T"). Do NOT invent or rename ids.',
+    '  • Return one entry per input source — same count, same ids.',
+    '  • reason: one short sentence (≤20 words).',
+  ].filter(Boolean).join('\n');
+}
+
+function buildUserPrompt(sources) {
+  // Hand the AI a JSON array — eliminates ambiguity about what the id is
+  const payload = sources.map((s) => {
+    const raw = s.source || {};
+    return {
+      id: s.id,
+      from: raw.from || s.from || '',
+      subject: raw.subject || s.subject || '',
+      categories: s.categories || (s.category ? [s.category] : []),
+      preview: (raw.content || '').slice(0, 400).replace(/\s+/g, ' '),
+    };
+  });
+
+  return `Score these ${sources.length} sources for brand fit. Return EXACTLY one score per source, using the "id" field verbatim from the input.\n\nInput sources:\n${JSON.stringify(payload, null, 2)}`;
+}
+
+module.exports = { filterSources, FILTER_SCHEMA, DEFAULT_THRESHOLD };

package/src/manager/libraries/email/generators/lib/image-host.js

@@ -0,0 +1,231 @@
+/**
+ * Asset host — uploads newsletter assets (per-section PNGs + the final
+ * rendered `newsletter.html`) to the public `itw-creative-works/newsletter-assets`
+ * GitHub repo as a single atomic commit.
+ *
+ * Returns publicly resolvable `raw.githubusercontent.com` URLs:
+ *   - Image URLs: embedded in the HTML via <img src=...> so Beehiiv / SendGrid /
+ *     any inbox can render them
+ *   - HTML URL: download link for manual paste into Beehiiv (and a stable
+ *     archive of every issue's final rendered form)
+ *
+ * Public-safety guarantees baked in:
+ *   - Only accepts PNG buffers for images — verified by magic-byte check
+ *   - HTML must be a non-empty string (no buffers, no other types)
+ *   - Path validated against a strict allowlist regex per file kind
+ *   - Repo / branch hardcoded (no env override) so a misconfigured caller
+ *     can't redirect uploads elsewhere
+ *   - One atomic commit per newsletter (Git Trees API)
+ *
+ * Pattern lifted from src/manager/routes/admin/post/post.js#commitAll —
+ * keep them in sync if the GitHub upload conventions change.
+ */
+const { Octokit } = require('@octokit/rest');
+
+const REPO_OWNER = 'itw-creative-works';
+const REPO_NAME  = 'newsletter-assets';
+const REPO_BRANCH = 'main';
+
+const RAW_BASE = `https://raw.githubusercontent.com/${REPO_OWNER}/${REPO_NAME}/${REPO_BRANCH}`;
+
+// `{brandId}/{campaignId}/section-N.png` — both ids are kebab/alphanumeric
+const IMAGE_PATH_REGEX = /^[a-z0-9-]+\/[A-Za-z0-9_-]+\/section-\d+\.png$/;
+// `{brandId}/{campaignId}/newsletter.html` — fixed file name, same folder
+const HTML_PATH_REGEX  = /^[a-z0-9-]+\/[A-Za-z0-9_-]+\/newsletter\.html$/;
+
+// PNG magic bytes: 89 50 4E 47 0D 0A 1A 0A
+const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+/**
+ * Upload newsletter assets (PNGs + optional HTML) as a single atomic commit.
+ *
+ * @param {object} args
+ * @param {Array<{png: Buffer}>} [args.images] - section images (only `png` is used).
+ *                                                Optional — pass an empty array or omit if
+ *                                                you only want to upload the HTML (rare).
+ * @param {string} [args.html] - The final rendered newsletter HTML. Uploaded as
+ *                                `{brandId}/{campaignId}/newsletter.html`.
+ * @param {string} args.brandId - lowercase brand slug (e.g. 'somiibo')
+ * @param {string} args.campaignId - Consumer-side `marketing-campaigns/{id}` Firestore doc ID.
+ *                                   Folder names use this verbatim — stable forever.
+ * @param {string} [args.subject] - Newsletter subject. Embedded in the commit message so
+ *                                  `git log` reads as a human-browseable history.
+ * @param {string} [args.commitMessage] - Full override of the default commit message
+ * @param {string} [args.token] - GitHub token (defaults to process.env.GITHUB_TOKEN)
+ * @param {object} [args.assistant] - logger
+ * @returns {Promise<{ urls: string[], paths: string[], htmlUrl?: string, htmlPath?: string, folderUrl: string, commitSha: string }>}
+ */
+async function uploadAssets({ images, html, brandId, campaignId, subject, commitMessage, token, assistant }) {
+  const hasImages = Array.isArray(images) && images.length > 0;
+  const hasHtml = typeof html === 'string' && html.length > 0;
+
+  if (!hasImages && !hasHtml) {
+    throw new Error('image-host: at least one of images[] or html must be provided');
+  }
+
+  validateBrandId(brandId);
+  validateCampaignId(campaignId);
+
+  const githubToken = token || process.env.GITHUB_TOKEN;
+
+  if (!githubToken) {
+    throw new Error('image-host: GITHUB_TOKEN env var (or token arg) is required');
+  }
+
+  const log = (msg) => assistant?.log ? assistant.log(`[image-host] ${msg}`) : null;
+
+  // 1. Build the list of files to commit. Validate each before we touch GitHub.
+  const files = [];
+
+  if (hasImages) {
+    for (let i = 0; i < images.length; i++) {
+      const img = images[i];
+      const path = `${brandId}/${campaignId}/section-${i + 1}.png`;
+
+      if (!IMAGE_PATH_REGEX.test(path)) {
+        throw new Error(`image-host: refusing to upload — invalid image path "${path}"`);
+      }
+
+      if (!Buffer.isBuffer(img.png)) {
+        throw new Error(`image-host: section ${i + 1} png is not a Buffer (got ${typeof img.png})`);
+      }
+
+      if (!img.png.slice(0, 8).equals(PNG_MAGIC)) {
+        throw new Error(`image-host: section ${i + 1} buffer is not a valid PNG (magic bytes mismatch)`);
+      }
+
+      files.push({
+        path,
+        contentBase64: img.png.toString('base64'),
+        kind: 'image',
+      });
+    }
+  }
+
+  if (hasHtml) {
+    const path = `${brandId}/${campaignId}/newsletter.html`;
+
+    if (!HTML_PATH_REGEX.test(path)) {
+      throw new Error(`image-host: refusing to upload — invalid html path "${path}"`);
+    }
+
+    files.push({
+      path,
+      contentBase64: Buffer.from(html, 'utf8').toString('base64'),
+      kind: 'html',
+    });
+  }
+
+  const imageCount = files.filter((f) => f.kind === 'image').length;
+  const summary = [
+    imageCount ? `${imageCount} PNG${imageCount === 1 ? '' : 's'}` : null,
+    hasHtml ? 'newsletter.html' : null,
+  ].filter(Boolean).join(' + ');
+
+  log(`uploading ${summary} to ${REPO_OWNER}/${REPO_NAME} → ${brandId}/${campaignId}/`);
+
+  const octokit = new Octokit({ auth: githubToken });
+
+  // 2. Get current main branch ref + tree
+  const { data: refData } = await octokit.rest.git.getRef({
+    owner: REPO_OWNER,
+    repo: REPO_NAME,
+    ref: `heads/${REPO_BRANCH}`,
+  });
+
+  const baseCommitSha = refData.object.sha;
+
+  const { data: baseCommit } = await octokit.rest.git.getCommit({
+    owner: REPO_OWNER,
+    repo: REPO_NAME,
+    commit_sha: baseCommitSha,
+  });
+
+  // 3. Create a blob per file
+  const treeItems = [];
+
+  for (const file of files) {
+    const { data: blob } = await octokit.rest.git.createBlob({
+      owner: REPO_OWNER,
+      repo: REPO_NAME,
+      content: file.contentBase64,
+      encoding: 'base64',
+    });
+
+    treeItems.push({
+      path: file.path,
+      mode: '100644',
+      type: 'blob',
+      sha: blob.sha,
+    });
+  }
+
+  // 4. Build new tree on top of base
+  const { data: newTree } = await octokit.rest.git.createTree({
+    owner: REPO_OWNER,
+    repo: REPO_NAME,
+    base_tree: baseCommit.tree.sha,
+    tree: treeItems,
+  });
+
+  // 5. Commit. Default message format: "[brand] campaignId — Subject" so
+  //    `git log` doubles as a human-readable index of the (opaque) folder names.
+  const defaultSubject = subject ? subject.trim() : `${files.length} newsletter asset${files.length === 1 ? '' : 's'}`;
+  const message = commitMessage || `[${brandId}] ${campaignId} — ${defaultSubject}`;
+
+  const { data: newCommit } = await octokit.rest.git.createCommit({
+    owner: REPO_OWNER,
+    repo: REPO_NAME,
+    message,
+    tree: newTree.sha,
+    parents: [baseCommitSha],
+  });
+
+  // 6. Update branch ref
+  await octokit.rest.git.updateRef({
+    owner: REPO_OWNER,
+    repo: REPO_NAME,
+    ref: `heads/${REPO_BRANCH}`,
+    sha: newCommit.sha,
+  });
+
+  // 7. Split the URL list by kind so callers can grab images + html independently.
+  const imageFiles = files.filter((f) => f.kind === 'image');
+  const htmlFile = files.find((f) => f.kind === 'html');
+
+  const result = {
+    urls: imageFiles.map((f) => `${RAW_BASE}/${f.path}`),
+    paths: imageFiles.map((f) => f.path),
+    folderUrl: `https://github.com/${REPO_OWNER}/${REPO_NAME}/tree/${REPO_BRANCH}/${brandId}/${campaignId}`,
+    commitSha: newCommit.sha,
+  };
+
+  if (htmlFile) {
+    result.htmlUrl = `${RAW_BASE}/${htmlFile.path}`;
+    result.htmlPath = htmlFile.path;
+  }
+
+  log(`committed ${newCommit.sha.slice(0, 7)} — folder: ${result.folderUrl}`);
+
+  return result;
+}
+
+function validateBrandId(brandId) {
+  if (!brandId || !/^[a-z0-9-]+$/.test(brandId)) {
+    throw new Error(`image-host: brandId must be lowercase alphanumeric+hyphens (got "${brandId}")`);
+  }
+}
+
+function validateCampaignId(campaignId) {
+  if (!campaignId || !/^[A-Za-z0-9_-]+$/.test(campaignId)) {
+    throw new Error(`image-host: campaignId must be alphanumeric/_/- (got "${campaignId}")`);
+  }
+}
+
+module.exports = {
+  uploadAssets,
+  REPO_OWNER,
+  REPO_NAME,
+  REPO_BRANCH,
+  RAW_BASE,
+};

package/src/manager/libraries/email/generators/lib/mjml-template.js

@@ -0,0 +1,83 @@
+/**
+ * MJML newsletter template — turns a structured newsletter into email-safe HTML.
+ *
+ * Dispatches by `newsletterConfig.template` into one of the layouts in
+ * `./templates/` and compiles the resulting MJML to HTML.
+ *
+ * Templates are pure builders — they take the structured data + theme tokens
+ * and return MJML. This file owns the compilation, UTM tagging, and brand
+ * resolution that's identical across templates.
+ */
+const mjml = require('mjml');
+
+const { resolveTemplate } = require('./templates/index.js');
+const { formatAddress } = require('./templates/shared.js');
+const { tagLinks } = require('../../utm');
+
+const DEFAULT_THEME = {
+  primaryColor:   '#5B5BFF',
+  secondaryColor: '#1E1E2A',
+  accentColor:    '#F6F7FB',
+  font:           'Inter, system-ui, sans-serif',
+};
+
+const DEFAULT_TEMPLATE = 'clean';
+
+/**
+ * Render the newsletter to email-safe HTML.
+ *
+ * @param {object} args
+ * @param {object} args.brand - { name, id, url }
+ * @param {object} args.newsletterConfig - marketing.beehiiv.content (theme, template, ...)
+ * @param {object} args.structure - Output from structure.js
+ * @param {string[]} args.imagePaths - One entry per section (URL or local path)
+ * @param {string} [args.campaign] - Used for UTM utm_campaign
+ * @param {string} [args.template] - Template override (otherwise newsletterConfig.template)
+ * @param {Array<object>} [args.sponsorships] - Brand-owned sponsorship promos to inject (merged with config)
+ * @returns {Promise<{mjml: string, html: string, template: string, errors: object[]}>}
+ */
+async function renderNewsletter({ brand, newsletterConfig, structure, imagePaths, campaign, template, sponsorships }) {
+  const theme = { ...DEFAULT_THEME, ...(newsletterConfig?.theme || {}) };
+  const brandName = brand?.name || 'Newsletter';
+  const brandUrl = brand?.url || '#';
+  const brandId = brand?.id || '';
+  const brandAddress = formatAddress(brand?.address);
+
+  const templateName = template || newsletterConfig?.template || DEFAULT_TEMPLATE;
+  const builder = resolveTemplate(templateName);
+
+  // Resolve sponsorships: per-call override beats per-campaign beats config defaults
+  const resolvedSponsorships = Array.isArray(sponsorships) && sponsorships.length
+    ? sponsorships
+    : (newsletterConfig?.sponsorships || []);
+
+  const mjmlString = builder.build({
+    structure,
+    imagePaths,
+    theme,
+    brandName,
+    brandUrl,
+    brandAddress,
+    sponsorships: resolvedSponsorships,
+    now: new Date(),
+  });
+
+  const compiled = await mjml(mjmlString, { validationLevel: 'soft' });
+
+  if (compiled.errors?.length) {
+    // Soft — log but don't throw. MJML often emits warnings that don't affect output.
+    // eslint-disable-next-line no-console
+    console.warn('MJML compilation warnings:', compiled.errors.map((e) => e.message));
+  }
+
+  const html = tagLinks(compiled.html, {
+    brandUrl,
+    brandId,
+    campaign: campaign || 'newsletter',
+    type: 'marketing',
+  });
+
+  return { mjml: mjmlString, html, template: templateName, errors: compiled.errors || [] };
+}
+
+module.exports = { renderNewsletter };