backend-manager 5.0.202 → 5.1.0

package/docs/testing.md

@@ -0,0 +1,129 @@
+# Testing
+
+## Running Tests
+
+```bash
+# Option 1: Two terminals
+npx mgr emulator  # Terminal 1 - keeps emulator running
+npx mgr test      # Terminal 2 - runs tests
+
+# Option 2: Single command (auto-starts emulator)
+npx mgr test
+```
+
+## Log Files
+
+BEM CLI commands automatically save all output to log files in `functions/` while still streaming to the console:
+- **`functions/serve.log`** — Output from `npx mgr serve` (Firebase serve)
+- **`functions/emulator.log`** — Full emulator output (Firebase emulator + Cloud Functions logs)
+- **`functions/test.log`** — Test runner output (when running against an existing emulator)
+- **`functions/logs.log`** — Cloud Function logs from `npx mgr logs:read` or `npx mgr logs:tail` (raw JSON for `read`, streaming text for `tail`)
+
+When `npx mgr test` starts its own emulator, logs go to `emulator.log` (since it delegates to the emulator command). When running against an already-running emulator, logs go to `test.log`.
+
+These files are overwritten on each run and are gitignored (`*.log`). Use them to search for errors, debug webhook pipelines, or review full function output after a test run.
+
+## Filtering Tests
+
+```bash
+npx mgr test rules/             # Run rules tests (both BEM and project)
+npx mgr test bem:rules/         # Only BEM's rules tests
+npx mgr test project:rules/     # Only project's rules tests
+npx mgr test user/ admin/       # Multiple paths
+```
+
+## Test Locations
+
+- **BEM core tests:** `test/`
+- **Project tests:** `functions/test/bem/`
+
+Use `bem:` or `project:` prefix to filter by source.
+
+## Test Types
+
+| Type | Use When | Behavior |
+|------|----------|----------|
+| Standalone | Single logical test | Runs once |
+| Suite (`type: 'suite'`) | Sequential dependent tests | Shared state, stops on failure |
+| Group (`type: 'group'`) | Multiple independent tests | Continues on failure |
+
+### Standalone Test
+
+```javascript
+module.exports = {
+  description: 'Test name',
+  auth: 'none',  // none, user, admin, premium-active, premium-expired
+  timeout: 10000,
+  async run({ http, assert, accounts, firestore, state, waitFor }) { },
+  async cleanup({ ... }) { },  // Optional
+};
+```
+
+### Suite (Sequential with Shared State)
+
+```javascript
+module.exports = {
+  description: 'Suite name',
+  type: 'suite',
+  tests: [
+    { name: 'step-1', async run({ state }) { state.value = 'shared'; } },
+    { name: 'step-2', async run({ state }) { /* state.value available */ } },
+  ],
+};
+```
+
+### Group (Independent Tests)
+
+```javascript
+module.exports = {
+  description: 'Group name',
+  type: 'group',
+  tests: [
+    { name: 'test-1', auth: 'admin', async run({ http, assert }) { } },
+    { name: 'test-2', auth: 'none', async run({ http, assert }) { } },
+  ],
+};
+```
+
+## Context Object
+
+| Property | Description |
+|----------|-------------|
+| `http` | HTTP client (`http.command()`, `http.as('admin').command()`) |
+| `assert` | Assertion helpers (see below) |
+| `accounts` | Test accounts `{ basic, admin, premium-active, ... }` |
+| `firestore` | Direct DB access (`get`, `set`, `delete`, `exists`) |
+| `state` | Shared state (suites only) |
+| `waitFor` | Polling helper `waitFor(condition, timeout, interval)` |
+
+## Assert Methods
+
+```javascript
+assert.ok(value, message)                      // Truthy
+assert.equal(a, b, message)                    // Strict equality
+assert.notEqual(a, b, message)                 // Not equal
+assert.deepEqual(a, b, message)                // Deep equality
+assert.match(value, /regex/, message)          // Regex match
+assert.isSuccess(response, message)            // Response success
+assert.isError(response, code, message)        // Response error with code
+assert.hasProperty(obj, 'path.to.prop', msg)   // Property exists
+assert.propertyEquals(obj, 'path', value, msg) // Property value
+assert.isType(value, 'string', message)        // Type check
+assert.contains(array, value, message)         // Array includes
+assert.inRange(value, min, max, message)       // Number range
+assert.fail(message)                           // Explicit fail
+```
+
+## Auth Levels
+
+`none`, `user`/`basic`, `admin`, `premium-active`, `premium-expired`
+
+## Key Test Files
+
+| File | Purpose |
+|------|---------|
+| `src/test/runner.js` | Test runner |
+| `test/` | BEM core tests |
+| `src/test/utils/assertions.js` | Assert helpers |
+| `src/test/utils/http-client.js` | HTTP client |
+| `src/test/test-accounts.js` | Test account definitions |

package/docs/usage-rate-limiting.md

@@ -0,0 +1,67 @@
+# Usage & Rate Limiting
+
+## Overview
+
+Usage is tracked per-metric (e.g., `requests`, `sponsorships`) with four fields:
+- `monthly`: Current month's count, reset on the 1st of each month by cron
+- `daily`: Current day's count, reset every day by cron
+- `total`: All-time count, never resets
+- `last`: Object with `id`, `timestamp`, `timestampUNIX` of the last usage event
+
+## Limits & Daily Caps
+
+Limits are always specified as **monthly** values in product config (e.g., `limits.requests = 100` means 100/month).
+
+By default, limits are enforced with **daily caps** to prevent users from burning their entire monthly quota in a single day. Two checks are applied:
+
+1. **Flat daily cap**: `ceil(monthlyLimit / daysInMonth)` — max uses per day
+   - e.g., 100/month in a 31-day month = `ceil(100/31)` = 4/day
+2. **Proportional monthly cap**: `ceil(monthlyLimit * dayOfMonth / daysInMonth)` — running total
+   - Prevents accumulating too much too fast even within daily limits
+   - e.g., Day 15 of a 30-day month with 100/month limit = max 50 used so far
+
+Products can opt out of daily caps by setting `rateLimit: 'monthly'` (default is `'daily'`):
+
+```json
+{
+  "id": "basic",
+  "limits": { "requests": 100 },
+  "rateLimit": "monthly"
+}
+```
+
+## Proxy Usage (setUser + Mirrors)
+
+Sometimes usage must be billed to a different user than the one making the request (e.g., anonymous visitors consuming an agent owner's credits). Use `setUser()` to swap the target and `addMirror()` / `setMirrors()` to write usage to additional Firestore docs:
+
+```js
+// Switch usage target to the agent owner (fetches their user doc)
+await usage.setUser(ownerUid);
+
+// Also write usage data to the agent doc
+usage.addMirror(`agents/${agentId}`);
+
+// Now validate, increment, and update all operate on the owner's data
+// update() writes to users/{ownerUid} AND agents/{agentId} in parallel
+await usage.validate('credits');
+usage.increment('credits');
+await usage.update();
+```
+
+**Methods:**
+- `setUser(uid)` — async, fetches `users/{uid}` from Firestore, replaces `self.user`, sets `useUnauthenticatedStorage = false`
+- `setMirrors(paths)` — sync, overwrites the mirror array with the given paths
+- `addMirror(path)` — sync, appends a single path to the mirror array
+
+Mirrors are write-only — `update()` writes `{ usage: self.user.usage }` (merge) to each mirror path. No reads are performed on mirrors.
+
+## Reset Schedule
+
+| Target | Frequency | What happens |
+|--------|-----------|-------------|
+| Local storage | Daily | Cleared entirely |
+| `usage` collection (unauthenticated) | Daily | Deleted entirely |
+| User doc `usage.*.daily` (authenticated) | Daily | Reset to 0 |
+| User doc `usage.*.monthly` (authenticated) | Monthly (1st) | Reset to 0 |
+
+The daily cron (`reset-usage.js`) runs at midnight UTC. It collects all users with non-zero counters across all metrics, then performs a single write per user to reset daily (and monthly on the 1st).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend-manager",
-  "version": "5.0.202",
+  "version": "5.1.0",
   "description": "Quick tools for developing Firebase functions",
   "main": "src/manager/index.js",
   "bin": {
@@ -49,15 +49,18 @@
     }
   },
   "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.140",
+    "@anthropic-ai/sdk": "^0.95.2",
     "@firebase/rules-unit-testing": "^5.0.1",
     "@google-cloud/firestore": "^7.11.6",
     "@google-cloud/pubsub": "^5.3.0",
     "@google-cloud/storage": "^7.19.0",
-    "@inquirer/prompts": "^8.4.2",
+    "@inquirer/prompts": "^8.4.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@octokit/rest": "^22.0.1",
+    "@resvg/resvg-js": "^2.6.2",
     "@sendgrid/mail": "^8.1.6",
-    "@sentry/node": "^10.52.0",
+    "@sentry/node": "^10.53.1",
     "body-parser": "^2.2.2",
     "busboy": "^1.6.0",
     "chalk": "^5.6.2",
@@ -75,6 +78,7 @@
     "mailchimp-api-v3": "^1.15.0",
     "markdown-it": "^14.1.1",
     "mime-types": "^3.0.2",
+    "mjml": "^5.2.1",
     "moment": "^2.30.1",
     "nanoid": "^5.1.11",
     "node-powertools": "^3.0.0",
@@ -87,7 +91,7 @@
     "wonderful-fetch": "^2.0.5",
     "wonderful-log": "^1.0.7",
     "wonderful-version": "^1.3.2",
-    "yaml": "^2.8.4",
+    "yaml": "^2.9.0",
     "yargs": "^18.0.0"
   },
   "devDependencies": {

package/src/defaults/CHANGELOG.md

@@ -0,0 +1,15 @@
+# CHANGELOG
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## Changelog Categories
+
+- `BREAKING` for breaking changes.
+- `Added` for new features.
+- `Changed` for changes in existing functionality.
+- `Deprecated` for soon-to-be removed features.
+- `Removed` for now removed features.
+- `Fixed` for any bug fixes.
+- `Security` in case of vulnerabilities.

package/src/defaults/CLAUDE.md

@@ -1,14 +1,16 @@
 # ========== Default Values ==========
 # Backend Manager (BEM) — consumer project
 
-> **Auto-managed file.** Everything between `# ========== Default Values ==========` and `# ========== Custom Values ==========` is owned by `backend-manager` and rewritten on every `npx mgr setup`. Put your own project-specific notes BELOW the `Custom Values` marker — that section is preserved verbatim across setups.
-
 ## Framework
 
 This project consumes **Backend Manager** (BEM) — a comprehensive framework for building modern Firebase Cloud Functions backends. BEM provides a single `Manager.init(exports, {...})` bootstrap that wires built-in functions (`bm_api`, auth events, cron jobs), helper classes (Assistant, User, Analytics, Usage, Middleware, Settings, Utilities, Metadata), payment processor integrations (Stripe / PayPal), Firestore-trigger pipelines, and a deploy/emulator/watch tooling pipeline.
 
-**Framework's own docs** (read these for deep-dives; both paths point to the same files, the absolute path works regardless of working directory):
-- Top-level overview: `/Users/ian/Developer/Repositories/ITW-Creative-Works/backend-manager/CLAUDE.md` (or `node_modules/backend-manager/CLAUDE.md`)
+## 🚨 READ THE FRAMEWORK DOCS FIRST
+
+**Before doing ANY work on this codebase, Claude MUST read the framework documentation — that is where the architecture, conventions, APIs, and gotchas live. Skipping these will result in solutions that conflict with framework patterns.**
+
+**Required reading:**
+- **`node_modules/backend-manager/CLAUDE.md`** — full framework reference (single comprehensive file; not yet split into per-subsystem docs)
 
 ## Quick start
 
@@ -66,6 +68,8 @@ After `Manager.init()`, the Manager instance exposes factory methods:
 
 Auth events, payment-webhook transitions, and cron jobs are wired automatically — hook into them by exporting from `functions/hooks/<area>/<event>.js`.
 
+<!-- Everything above this marker is owned by the framework and rewritten on every `npx mgr setup`. Add your project-specific notes below — they are preserved across setups. -->
+
 # ========== Custom Values ==========
 
 ## Project-specific notes

package/src/defaults/docs/README.md

@@ -0,0 +1,17 @@
+# Project docs
+
+Per-subsystem deep references live here. Keep `CLAUDE.md` short — it should read as a **table of contents** that points at files in this directory.
+
+## Pattern
+
+When you find yourself adding more than a paragraph to `CLAUDE.md`, create a new `docs/<topic>.md` instead and link to it from `CLAUDE.md`. Goal: the project's `CLAUDE.md` stays under ~250 lines.
+
+Examples of good `docs/*.md` topics:
+- Subsystem deep-dives (one per area of the codebase)
+- Architectural decisions / "why we built it this way"
+- Defaults tables, behavior matrices, edge cases
+- Setup walkthroughs that don't belong in `README.md`
+
+## See also
+
+`node_modules/backend-manager/CLAUDE.md` is the framework's own overview.

package/src/defaults/test/README.md

@@ -0,0 +1,33 @@
+# Project tests
+
+Drop your project test suites here. The framework auto-runs them alongside its own when you run `npx mgr test`.
+
+## Layout
+
+Match the framework's layout — Backend Manager's test runner discovers files by the directory they sit in. Mirror the same per-area split as the framework's own `test/` (see `node_modules/backend-manager/test/`):
+
+| Directory | Use for |
+|---|---|
+| `test/routes/` | Custom HTTP route handlers (`functions/routes/<verb>/<path>.js`) |
+| `test/events/` | Pub/Sub / Firestore-trigger handlers |
+| `test/helpers/` | Shared test utilities for your project |
+| `test/fixtures/` | Static test data (JSON, sample docs) |
+| `test/_init/` | Per-suite setup (Firestore seed data, user accounts) |
+
+Tests run inside the Firebase emulator. Use the BEM helpers (`assistant`, admin SDK, fixture loaders) instead of mocking — `npx mgr emulator` boots the same environment the tests run against.
+
+## Quick example
+
+```js
+// test/routes/hello.test.js
+module.exports = {
+  'GET /hello returns ok': async ({ http }) => {
+    const res = await http.get('hello');
+    if (res.status !== 200) throw new Error('expected 200');
+  },
+};
+```
+
+## See also
+
+The framework's own test suites at `node_modules/backend-manager/test/` are the canonical reference for how each layer is structured.

package/src/manager/events/cron/daily/marketing-newsletter-generate.js

@@ -2,13 +2,29 @@
  * Newsletter pre-generation cron job
  *
  * Runs daily. Looks for generator campaigns (e.g., _recurring-newsletter)
- * with sendAt within the next 24 hours. Generates content via AI and creates
- * a NEW standalone pending campaign with the real content.
+ * with sendAt within the next 24 hours. For each due campaign it runs the
+ * FULL pipeline:
+ *
+ *   1. Fetch + filter sources from parent server (per brand category set)
+ *   2. AI authors structured content (subject, sections/dispatches, signoff, ...)
+ *   3. AI authors per-section SVG, rasterized to PNG
+ *   4. Upload PNGs to itw-creative-works/newsletter-assets/{brandId}/{newId}/
+ *      and render MJML → email-safe HTML embedding those URLs
+ *   5. Upload the rendered newsletter.html into the same folder so the issue
+ *      has a browseable, downloadable archive (and a paste-into-Beehiiv URL)
+ *   6. Create a NEW pending campaign doc with the generated content + asset URLs
+ *   7. Advance the recurring template's sendAt to the next occurrence
  *
  * The generated campaign appears on the calendar for review.
  * The frequent cron picks it up and sends it when sendAt is due.
  *
- * After generating, advances the recurring doc's sendAt to the next occurrence.
+ * Generated doc shape (marketing-campaigns/{newId}):
+ *   {
+ *     settings:        { ...generated content, subject, contentHtml, ... },
+ *     assets:          { folderUrl, htmlUrl, imageUrls, campaignId },
+ *     meta:            { telemetry — tokens, cost, durations, source scores },
+ *     type, sendAt, status: 'pending', generatedFrom, metadata
+ *   }
  *
  * Runs on bm_cronDaily.
  */
@@ -55,21 +71,38 @@ module.exports = async ({ Manager, assistant, libraries }) => {
 
     assistant.log(`Generating content for ${campaignId} (${generator}): ${settings.name}`);
 
-    // Run the generator
-    const generated = await generators[generator].generate(Manager, assistant, settings);
+    // Reserve the new doc ID UP FRONT so the generator can use it as the
+    // GitHub folder name. The asset URLs (raw.githubusercontent.com/.../{newId}/...)
+    // get baked into the rendered HTML, and we want those URLs to match the
+    // Firestore doc that hosts the campaign. Stable, predictable, browseable.
+    const newId = pushid();
+
+    // Run the generator with imageHost forced to 'github' (production cron
+    // path always uploads — that's what "production" means here) and the
+    // campaignId pinned so all assets land in marketing-campaigns/{newId}/'s
+    // matching folder.
+    const generated = await generators[generator].generate(Manager, assistant, settings, {
+      campaignId: newId,
+      imageHost: 'github',
+    });
 
     if (!generated) {
       assistant.log(`Generator "${generator}" returned no content for ${campaignId}, skipping`);
       continue;
     }
 
-    // Create a new standalone campaign with the generated content
-    const newId = pushid();
     const nowISO = new Date().toISOString();
     const nowUNIX = Math.round(Date.now() / 1000);
 
+    // Strip non-serializable fields out of the generator's return before
+    // writing to Firestore. `images` is an array of PNG Buffers (not safe
+    // to persist) and `mjml` is a raw template string that pollutes the doc.
+    const { images: _images, mjml: _mjml, assets, meta, ...campaignSettings } = generated;
+
     await admin.firestore().doc(`marketing-campaigns/${newId}`).set({
-      settings: generated,
+      settings: campaignSettings,
+      assets: assets || null,   // { folderUrl, htmlUrl, imageUrls, campaignId } or null
+      meta:   meta   || null,   // tokens, cost, durations, source scores
       type,
       sendAt: data.sendAt,
       status: 'pending',
@@ -81,6 +114,13 @@ module.exports = async ({ Manager, assistant, libraries }) => {
     });
 
     assistant.log(`Created campaign ${newId} from generator ${campaignId}: "${generated.subject}"`);
+    if (assets?.htmlUrl) {
+      assistant.log(`  HTML:   ${assets.htmlUrl}`);
+      assistant.log(`  Folder: ${assets.folderUrl}`);
+    }
+    if (assets?.beehiivPostId) {
+      assistant.log(`  Beehiiv: draft post ${assets.beehiivPostId}`);
+    }
 
     // Advance the recurring doc's sendAt to the next occurrence
     if (recurrence) {

package/src/manager/functions/core/actions/api/admin/create-post.js

@@ -55,20 +55,8 @@ Module.prototype.main = function () {
         return reject(assistant.errorify(`Missing required parameter: body`, {code: 400}));
       }
 
-      // Fix required values
-      payload.data.payload.url = payload.data.payload.url
-        // Replace blog/
-        .replace(/blog\//ig, '')
-        // Remove leading and trailing slashes
-        .replace(/^\/|\/$/g, '')
-        // Replace anything that's not a letter or number with a hyphen
-        .replace(/[^a-zA-Z0-9]/g, '-')
-        // Remove multiple hyphens
-        .replace(/-+/g, '-')
-        // Remove leading and trailing hyphens
-        .replace(/^-+|-+$/g, '')
-        // Lowercase
-        .toLowerCase();
+      // Fix required values — strip blog/ prefix then slugify (slugify handles slashes/special chars)
+      payload.data.payload.url = Manager.Utilities().slugify(payload.data.payload.url.replace(/blog\//ig, ''));
 
       // Fix body
       payload.data.payload.body = payload.data.payload.body
@@ -207,7 +195,7 @@ Module.prototype.downloadImage = function (src, alt) {
 
   return new Promise(async function(resolve, reject) {
     // Log
-    const hyphenated = hyphenate(alt);
+    const hyphenated = Manager.Utilities().slugify(alt);
 
     // Log
     assistant.log(`downloadImage(): src=${src}, alt=${alt}, hyphenated=${hyphenated}`);
@@ -370,16 +358,4 @@ function formatClone(payload) {
   return payload;
 }
 
-function hyphenate(s) {
-  return s
-    // Remove everything that is not a letter or a number
-    .replace(/[^a-zA-Z0-9]/g, '-')
-    // Replace multiple hyphens with a single hyphen
-    .replace(/-+/g, '-')
-    // Remove leading and trailing hyphens
-    .replace(/^-|-$/g, '')
-    // Lowercase
-    .toLowerCase();
-}
-
 module.exports = Module;

package/src/manager/helpers/settings.js

@@ -47,19 +47,38 @@ Settings.prototype.resolve = function (assistant, schema, settings, options) {
     const method = (assistant?.request?.method || '').toLowerCase();
     const methodFile = `${method}.js`;
     const schemaFile = options.schema.replace('.js', '');
-    let schemaPath;
 
-    // First try method-specific schema (e.g., test/get.js, test/post.js)
     const methodSchemaPath = path.resolve(options.dir, `${schemaFile}/${methodFile}`);
+    const indexSchemaPath = path.resolve(options.dir, `${schemaFile}/index.js`);
+
+    // Helper: only fall back when THIS specific file is missing.
+    // If the file exists but throws (syntax error, runtime error, etc.) we re-throw
+    // so the real problem surfaces instead of being masked by a misleading fallback.
+    const isMissingModule = (err, expectedPath) => err
+      && err.code === 'MODULE_NOT_FOUND'
+      && typeof err.message === 'string'
+      && err.message.includes(expectedPath);
 
     try {
       schema = loadSchema(assistant, methodSchemaPath);
       assistant.log(`Settings.resolve(): Loaded method-specific schema: ${schemaFile}/${methodFile}`);
-    } catch (e) {
-      // Fallback to main schema if method-specific doesn't exist
-      schemaPath = path.resolve(options.dir, `${schemaFile}/index.js`);
-      schema = loadSchema(assistant, schemaPath);
-      assistant.log(`Settings.resolve(): Method-specific schema not found, using main schema fallback`);
+    } catch (methodErr) {
+      if (!isMissingModule(methodErr, methodSchemaPath)) {
+        throw methodErr;
+      }
+
+      try {
+        schema = loadSchema(assistant, indexSchemaPath);
+        assistant.log(`Settings.resolve(): Method-specific schema not found, using main schema fallback`);
+      } catch (indexErr) {
+        if (!isMissingModule(indexErr, indexSchemaPath)) {
+          throw indexErr;
+        }
+        throw assistant.errorify(
+          `No schema for ${method.toUpperCase()} request: expected ${schemaFile}/${methodFile} or ${schemaFile}/index.js`,
+          {code: 500},
+        );
+      }
     }
   }
 

package/src/manager/helpers/utilities.js

@@ -468,6 +468,27 @@ Utilities.prototype.get = function (docPath, options) {
   });
 };
 
+/**
+ * Convert a string into a URL-safe slug.
+ * Strips all non-alphanumeric characters, collapses runs of hyphens, lowercases.
+ * Canonical slug builder — share this between BEM admin/post and any consumer
+ * that needs to predict the resulting URL (e.g. sponsorship platform).
+ *
+ * @param {string} input - The string to slugify
+ * @returns {string} URL-safe slug (or empty string for non-string input)
+ */
+Utilities.prototype.slugify = function (input) {
+  if (typeof input !== 'string') {
+    return '';
+  }
+
+  return input
+    .replace(/[^a-zA-Z0-9]/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .toLowerCase();
+};
+
 /**
  * Sanitize input by stripping HTML tags and trimming strings.
  * Accepts any data type — walks objects/arrays recursively.

package/src/manager/index.js

@@ -558,7 +558,7 @@ Manager.prototype.Email = function (assistant) {
 
 Manager.prototype.AI = function (assistant, key) {
   const self = this;
-  self.libraries.AI = self.libraries.AI || require('./libraries/openai.js');
+  self.libraries.AI = self.libraries.AI || require('./libraries/ai/index.js');
   return new self.libraries.AI(assistant, key);
 };