backend-manager 5.9.16 → 5.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/ghostii.md +63 -5
- package/package.json +1 -1
- package/src/manager/events/cron/daily/blog-auto-publisher.js +23 -398
- package/src/manager/libraries/content/ghostii.js +2 -1
- package/src/manager/libraries/content/source-resolver.js +397 -0
- package/src/manager/libraries/email/generators/newsletter.js +18 -46
- package/src/manager/routes/admin/post/post.js +1 -0
- package/src/manager/routes/admin/post/templates/post.html +1 -0
- package/src/manager/routes/content/post/get.js +1 -0
- package/src/test/fixtures/firebase-project/.temp/test-mode.json +1 -1
- package/src/test/fixtures/firebase-project/database-debug.log +8 -8
- package/src/test/fixtures/firebase-project/firestore-debug.log +56 -56
- package/src/test/fixtures/firebase-project/pubsub-debug.log +3 -3
- package/test/email/newsletter-generate.js +65 -50
package/docs/ghostii.md
CHANGED
|
@@ -27,9 +27,10 @@ trackContentSource() -- Firestore (for $feed:, $parent sources)
|
|
|
27
27
|
|
|
28
28
|
| File | Purpose |
|
|
29
29
|
|---|---|
|
|
30
|
+
| `src/manager/libraries/content/source-resolver.js` | **Shared SSOT**: prompt templates, anti-traceability rules, feed/parent resolution, Firestore tracking, fallback chain. Used by both blog + newsletter. |
|
|
30
31
|
| `src/manager/libraries/content/ghostii.js` | `writeArticle()`, `publishArticle()`, `blocksToPost()` -- Ghostii API client + post transform |
|
|
31
32
|
| `src/manager/libraries/content/feed-parser.js` | `parseFeed()`, `extractArticleContent()` -- RSS/Atom/JSON parser + article extractor |
|
|
32
|
-
| `src/manager/events/cron/daily/blog-auto-publisher.js` | Daily cron: source
|
|
33
|
+
| `src/manager/events/cron/daily/blog-auto-publisher.js` | Daily cron: imports from source-resolver, manages harvest loop + provider dispatch |
|
|
33
34
|
| `src/manager/routes/admin/post/post.js` | Publishing endpoint: image download + resize, GitHub commit |
|
|
34
35
|
|
|
35
36
|
## Source Types
|
|
@@ -145,11 +146,68 @@ When a `$feed:` or `$parent` source provides extracted article text, it's passed
|
|
|
145
146
|
|
|
146
147
|
- **`extractArticleContent(url)`** -- Fetch URL, extract readable text from `<article>` -> `<main>` -> `<body>`, strip scripts/styles/nav/footer/header/aside, normalize whitespace, truncate to 14KB.
|
|
147
148
|
|
|
149
|
+
## Prompt Templates (source-resolver.js)
|
|
150
|
+
|
|
151
|
+
Both blog and newsletter use shared prompt constants from `source-resolver.js`:
|
|
152
|
+
|
|
153
|
+
- **`PROMPT_SOURCE`** — used when a `$feed:` or `$parent` source is resolved. Directs the AI to cover the SAME topic with a different angle, title, and structure. Includes anti-traceability rules.
|
|
154
|
+
- **`PROMPT`** — used for `$brand`, URL, and text sources. Open-ended topic generation.
|
|
155
|
+
- **`ANTI_TRACEABILITY`** — shared rules: never name the source publication, paraphrase all data. Embedded in `PROMPT_SOURCE`, also used by the newsletter's system prompt.
|
|
156
|
+
|
|
157
|
+
The brand's `instructions` field is injected into both templates and can override the default behavior (e.g. a brand that WANTS genericized output can say so in instructions).
|
|
158
|
+
|
|
148
159
|
## Tests
|
|
149
160
|
|
|
161
|
+
### Unit tests (fast, free)
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
npx mgr test mgr:helpers/content/feed-parser # 32 tests: RSS/Atom/JSON parsing, edge cases
|
|
165
|
+
npx mgr test mgr:helpers/content/ghostii-write-article # 15 tests: override pass-through, sourceContent
|
|
166
|
+
npx mgr test mgr:helpers/content/blog-auto-publisher # 25+ tests: source detection, feed processing, tracking
|
|
167
|
+
npx mgr test mgr:helpers/content/ghostii-blocks # 8 tests: blocksToPost()
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Blog generation (full AI pipeline)
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
# Config check only (fast, free)
|
|
174
|
+
npx mgr test mgr:content/blog-generate
|
|
175
|
+
|
|
176
|
+
# Full AI pipeline — generates article, does NOT publish
|
|
177
|
+
BLOG_NO_PUBLISH=1 TEST_EXTENDED_MODE=1 npx mgr test mgr:content/blog-generate
|
|
178
|
+
|
|
179
|
+
# Override source type
|
|
180
|
+
BLOG_SOURCE='$feed:https://feeds.arstechnica.com/arstechnica/index' \
|
|
181
|
+
BLOG_NO_PUBLISH=1 TEST_EXTENDED_MODE=1 npx mgr test mgr:content/blog-generate
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
| Env var | Description |
|
|
185
|
+
|---|---|
|
|
186
|
+
| `TEST_EXTENDED_MODE=1` | Enable full AI pipeline (costs ~$0.10-0.50 per run) |
|
|
187
|
+
| `BLOG_NO_PUBLISH=1` | Generate but do NOT publish to website repo |
|
|
188
|
+
| `BLOG_SOURCE=<type>` | Override source: `$brand`, `$parent`, `$feed:<url>`, URL, or text |
|
|
189
|
+
| `BLOG_OPEN=1` | Auto-open generated article (macOS only) |
|
|
190
|
+
|
|
191
|
+
### Newsletter generation (full AI pipeline)
|
|
192
|
+
|
|
150
193
|
```bash
|
|
151
|
-
|
|
152
|
-
npx mgr test
|
|
153
|
-
|
|
154
|
-
|
|
194
|
+
# Fixture render only (fast, free)
|
|
195
|
+
npx mgr test mgr:email/newsletter-generate
|
|
196
|
+
|
|
197
|
+
# Full AI pipeline with feed source — generates newsletter, does NOT publish article
|
|
198
|
+
NEWSLETTER_SOURCE='$feed:https://feeds.arstechnica.com/arstechnica/index' \
|
|
199
|
+
NEWSLETTER_NO_IMAGES=1 TEST_EXTENDED_MODE=1 npx mgr test mgr:email/newsletter-generate
|
|
155
200
|
```
|
|
201
|
+
|
|
202
|
+
| Env var | Description |
|
|
203
|
+
|---|---|
|
|
204
|
+
| `TEST_EXTENDED_MODE=1` | Enable full AI pipeline |
|
|
205
|
+
| `NEWSLETTER_SOURCE=<type>` | Override source: `$feed:<url>`, `$parent`. Bypasses parent-server fetch, lets the generator's resolver run. |
|
|
206
|
+
| `NEWSLETTER_NO_IMAGES=1` | Skip image generation (fast iteration on copy) |
|
|
207
|
+
| `NEWSLETTER_CREATE_ARTICLE=1` | Publish linked blog article (OFF by default) |
|
|
208
|
+
| `NEWSLETTER_FIXTURE=<name>` | Load specific fixture for render test |
|
|
209
|
+
| `NEWSLETTER_TEMPLATE=<name>` | Override layout template |
|
|
210
|
+
| `NEWSLETTER_OPEN=1` | Auto-open preview (macOS only) |
|
|
211
|
+
| `NEWSLETTER_PEEK=1` | List ready parent sources, don't generate |
|
|
212
|
+
| `NEWSLETTER_SOURCE_ID=<id>` | Test specific parent source |
|
|
213
|
+
| `NEWSLETTER_CAMPAIGN_ID=<id>` | Override campaign folder name |
|
package/package.json
CHANGED
|
@@ -16,87 +16,49 @@
|
|
|
16
16
|
* All feed/parent sources are tracked in Firestore (`content-sources`) so the
|
|
17
17
|
* same article is never processed twice. When a feed is unreachable or
|
|
18
18
|
* exhausted, the entry falls back to $brand behavior.
|
|
19
|
+
*
|
|
20
|
+
* Source resolution, tracking, and prompt constants live in the shared
|
|
21
|
+
* source-resolver library (src/manager/libraries/content/source-resolver.js).
|
|
19
22
|
*/
|
|
20
|
-
const crypto = require('crypto');
|
|
21
|
-
const fetch = require('wonderful-fetch');
|
|
22
23
|
const powertools = require('node-powertools');
|
|
23
24
|
const moment = require('moment');
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
`;
|
|
39
|
-
|
|
40
|
-
const PROMPT_SOURCE = `
|
|
41
|
-
Company: {brand.brand.name}: {brand.brand.description}
|
|
42
|
-
Date: {date}
|
|
43
|
-
Instructions: {instructions}
|
|
44
|
-
Tone: {tone}
|
|
45
|
-
Categories: {categories}
|
|
46
|
-
Keywords: {keywords}
|
|
47
|
-
|
|
48
|
-
Write an original article inspired by and referencing this source material.
|
|
49
|
-
Do NOT copy the source — use it as context and inspiration for a unique take.
|
|
50
|
-
Source title: {sourceTitle}
|
|
51
|
-
`;
|
|
52
|
-
|
|
53
|
-
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
|
|
54
|
-
const FEED_PREFIX = '$feed:';
|
|
55
|
-
const CONTENT_SOURCES_COLLECTION = 'content-sources';
|
|
56
|
-
|
|
57
|
-
// State
|
|
25
|
+
|
|
26
|
+
const {
|
|
27
|
+
PROMPT_SOURCE,
|
|
28
|
+
PROMPT,
|
|
29
|
+
FEED_PREFIX,
|
|
30
|
+
processFeedSource,
|
|
31
|
+
processParentSource,
|
|
32
|
+
trackContentSource,
|
|
33
|
+
contentSourceHash,
|
|
34
|
+
getProcessedItemIds,
|
|
35
|
+
getURLContent,
|
|
36
|
+
isURL,
|
|
37
|
+
} = require('../../../libraries/content/source-resolver.js');
|
|
38
|
+
|
|
58
39
|
let postId;
|
|
59
40
|
|
|
60
|
-
/**
|
|
61
|
-
* Blog Auto Publisher cron job
|
|
62
|
-
*
|
|
63
|
-
* Automatically generates and publishes blog posts using the configured platform provider.
|
|
64
|
-
*/
|
|
65
41
|
module.exports = async ({ Manager, assistant, context, libraries }) => {
|
|
66
|
-
// Gate on blog.enabled
|
|
67
42
|
if (!Manager.config.blog?.enabled) {
|
|
68
43
|
assistant.log('Blog auto-publisher disabled in config');
|
|
69
44
|
return;
|
|
70
45
|
}
|
|
71
46
|
|
|
72
|
-
// Log
|
|
73
47
|
assistant.log('Starting...');
|
|
74
48
|
|
|
75
|
-
// Set post ID
|
|
76
49
|
postId = moment().unix();
|
|
77
50
|
|
|
78
|
-
// Build brand config from local config
|
|
79
51
|
const brandConfig = buildBrandConfig(Manager.config);
|
|
80
|
-
|
|
81
|
-
// Log
|
|
82
52
|
assistant.log('Brand config', brandConfig);
|
|
83
53
|
|
|
84
|
-
// Get blog config
|
|
85
54
|
const blog = Manager.config.blog;
|
|
86
|
-
|
|
87
|
-
// Get content entries (array or single object)
|
|
88
55
|
const contentArray = powertools.arrayify(blog.content);
|
|
89
|
-
|
|
90
|
-
// Get admin for Firestore tracking
|
|
91
56
|
const { admin } = libraries;
|
|
92
57
|
|
|
93
|
-
// Load the provider based on blog.platform
|
|
94
58
|
const platform = blog.platform || 'ghostii';
|
|
95
59
|
const provider = require(`../../../libraries/content/${platform}.js`);
|
|
96
60
|
|
|
97
|
-
// Loop through each content entry
|
|
98
61
|
for (const entry of contentArray) {
|
|
99
|
-
// Normalize entry fields
|
|
100
62
|
entry.quantity = entry.quantity || 0;
|
|
101
63
|
entry.sources = randomize(entry.sources || []);
|
|
102
64
|
entry.links = randomize(entry.links || []);
|
|
@@ -109,60 +71,45 @@ module.exports = async ({ Manager, assistant, context, libraries }) => {
|
|
|
109
71
|
entry.postPath = entry.postPath || platform;
|
|
110
72
|
entry.overrides = entry.overrides || {};
|
|
111
73
|
|
|
112
|
-
// Resolve brand data for this entry
|
|
113
74
|
if (entry.brand && entry.brandUrl) {
|
|
114
|
-
// Cross-brand: fetch from the other project's /brand endpoint
|
|
115
75
|
entry.brand = await fetchRemoteBrand(entry.brandUrl).catch((e) => e);
|
|
116
|
-
|
|
117
76
|
if (entry.brand instanceof Error) {
|
|
118
77
|
assistant.error('Error fetching remote brand data', entry.brand);
|
|
119
78
|
continue;
|
|
120
79
|
}
|
|
121
80
|
} else {
|
|
122
|
-
// Same-brand: use local config
|
|
123
81
|
entry.brand = brandConfig;
|
|
124
82
|
}
|
|
125
83
|
|
|
126
|
-
// Log
|
|
127
84
|
assistant.log(`Entry (brand=${entry.brand.brand.id})`, entry);
|
|
128
85
|
|
|
129
|
-
// Quit if quantity is zero or no sources
|
|
130
86
|
if (!entry.quantity || !entry.sources.length) {
|
|
131
87
|
assistant.log('Quitting because quantity is 0 or no sources');
|
|
132
88
|
continue;
|
|
133
89
|
}
|
|
134
90
|
|
|
135
|
-
// Quit if the chance is not met
|
|
136
91
|
const chance = Math.random();
|
|
137
92
|
if (chance > entry.chance) {
|
|
138
93
|
assistant.log(`Quitting because the chance is not met (${chance} <= ${entry.chance})`);
|
|
139
94
|
continue;
|
|
140
95
|
}
|
|
141
96
|
|
|
142
|
-
// Harvest articles
|
|
143
97
|
const result = await harvest(assistant, entry, admin, provider, Manager).catch((e) => e);
|
|
144
98
|
if (result instanceof Error) {
|
|
145
99
|
throw result;
|
|
146
100
|
}
|
|
147
101
|
|
|
148
|
-
// Log
|
|
149
102
|
assistant.log('Finished!', result);
|
|
150
103
|
}
|
|
151
104
|
};
|
|
152
105
|
|
|
153
|
-
/**
|
|
154
|
-
* Build brand config from Manager.config (same shape as /brand endpoint response)
|
|
155
|
-
*/
|
|
156
106
|
function buildBrandConfig(config) {
|
|
157
107
|
const { buildPublicConfig } = require(require('path').join(__dirname, '..', '..', '..', 'routes', 'brand', 'get.js'));
|
|
158
|
-
|
|
159
108
|
return buildPublicConfig(config);
|
|
160
109
|
}
|
|
161
110
|
|
|
162
|
-
/**
|
|
163
|
-
* Fetch brand data from a remote BEM project's /brand endpoint
|
|
164
|
-
*/
|
|
165
111
|
function fetchRemoteBrand(brandUrl) {
|
|
112
|
+
const fetch = require('wonderful-fetch');
|
|
166
113
|
return fetch(`${brandUrl}/backend-manager/brand`, {
|
|
167
114
|
timeout: 120000,
|
|
168
115
|
tries: 3,
|
|
@@ -173,33 +120,26 @@ function fetchRemoteBrand(brandUrl) {
|
|
|
173
120
|
async function harvest(assistant, entry, admin, provider, Manager) {
|
|
174
121
|
const date = moment().format('MMMM YYYY');
|
|
175
122
|
|
|
176
|
-
// Log
|
|
177
123
|
assistant.log(`harvest(): Starting ${entry.brand.brand.id}...`);
|
|
178
124
|
|
|
179
|
-
// Process the number of sources in the entry
|
|
180
125
|
for (let index = 0; index < entry.quantity; index++) {
|
|
181
126
|
const source = powertools.random(entry.sources);
|
|
182
127
|
|
|
183
|
-
// Log
|
|
184
128
|
assistant.log(`harvest(): Processing ${index + 1}/${entry.quantity}`, source);
|
|
185
129
|
|
|
186
|
-
// Resolve the source into a prompt description + optional sourceContent
|
|
187
130
|
const resolved = await resolveSource(assistant, source, entry, admin, Manager).catch((e) => e);
|
|
188
131
|
if (resolved instanceof Error) {
|
|
189
132
|
assistant.error('harvest(): Error resolving source', resolved);
|
|
190
133
|
break;
|
|
191
134
|
}
|
|
192
135
|
|
|
193
|
-
// Log
|
|
194
136
|
assistant.log('harvest(): Resolved source', resolved);
|
|
195
137
|
|
|
196
|
-
// Use keywords from entry if overrides.keywords is not set
|
|
197
138
|
const overrides = { ...entry.overrides };
|
|
198
139
|
if (!overrides.keywords && entry.keywords.length) {
|
|
199
140
|
overrides.keywords = entry.keywords;
|
|
200
141
|
}
|
|
201
142
|
|
|
202
|
-
// Request article from provider
|
|
203
143
|
const article = await provider.writeArticle({
|
|
204
144
|
brand: entry.brand,
|
|
205
145
|
description: resolved.description,
|
|
@@ -212,26 +152,23 @@ async function harvest(assistant, entry, admin, provider, Manager) {
|
|
|
212
152
|
break;
|
|
213
153
|
}
|
|
214
154
|
|
|
215
|
-
// Log
|
|
216
155
|
assistant.log('harvest(): Article', article);
|
|
217
156
|
|
|
218
|
-
// Upload post to blog
|
|
219
157
|
const uploadedPost = await provider.publishArticle(assistant, {
|
|
220
158
|
brand: entry.brand,
|
|
221
159
|
article,
|
|
222
160
|
id: postId++,
|
|
223
161
|
author: entry.author,
|
|
224
162
|
postPath: entry.postPath,
|
|
163
|
+
source: resolved.trackingData?.url || null,
|
|
225
164
|
}).catch((e) => e);
|
|
226
165
|
if (uploadedPost instanceof Error) {
|
|
227
166
|
assistant.error('harvest(): Error uploading post to blog', uploadedPost);
|
|
228
167
|
break;
|
|
229
168
|
}
|
|
230
169
|
|
|
231
|
-
// Log
|
|
232
170
|
assistant.log('harvest(): Uploaded post', uploadedPost);
|
|
233
171
|
|
|
234
|
-
// Track content source in Firestore
|
|
235
172
|
if (resolved.trackingData && admin) {
|
|
236
173
|
await trackContentSource(admin, {
|
|
237
174
|
...resolved.trackingData,
|
|
@@ -245,22 +182,9 @@ async function harvest(assistant, entry, admin, provider, Manager) {
|
|
|
245
182
|
}
|
|
246
183
|
}
|
|
247
184
|
|
|
248
|
-
/**
|
|
249
|
-
* Resolve a source entry into { description, sourceContent, trackingData? }.
|
|
250
|
-
*
|
|
251
|
-
* Source types:
|
|
252
|
-
* '$brand' — generic topic prompt, no sourceContent
|
|
253
|
-
* '$feed:<url>' — RSS/Atom feed: pick unprocessed article, extract content
|
|
254
|
-
* '$parent' — fetch sources from parent server without claiming
|
|
255
|
-
* URL — fetch page content as prompt seed
|
|
256
|
-
* text — use directly as prompt seed
|
|
257
|
-
*
|
|
258
|
-
* Feed/parent failures fall back to $brand behavior.
|
|
259
|
-
*/
|
|
260
185
|
async function resolveSource(assistant, source, entry, admin, Manager) {
|
|
261
186
|
const date = moment().format('MMMM YYYY');
|
|
262
187
|
|
|
263
|
-
// Build template vars for prompts
|
|
264
188
|
const templateVars = {
|
|
265
189
|
...entry,
|
|
266
190
|
instructions: entry.instructions,
|
|
@@ -305,13 +229,14 @@ async function resolveSource(assistant, source, entry, admin, Manager) {
|
|
|
305
229
|
if (source === '$parent') {
|
|
306
230
|
assistant.log('resolveSource(): Processing $parent source');
|
|
307
231
|
|
|
308
|
-
const
|
|
232
|
+
const parentResults = await processParentSource(assistant, entry, admin, Manager).catch((e) => e);
|
|
309
233
|
|
|
310
|
-
if (
|
|
234
|
+
if (parentResults instanceof Error || !parentResults?.length) {
|
|
311
235
|
assistant.log('resolveSource(): Parent source failed or exhausted, falling back to $brand');
|
|
312
236
|
return resolveSource(assistant, '$brand', entry, admin, Manager);
|
|
313
237
|
}
|
|
314
238
|
|
|
239
|
+
const parentResult = parentResults[0];
|
|
315
240
|
const description = powertools.template(PROMPT_SOURCE, {
|
|
316
241
|
...templateVars,
|
|
317
242
|
sourceTitle: parentResult.title,
|
|
@@ -367,311 +292,11 @@ async function resolveSource(assistant, source, entry, admin, Manager) {
|
|
|
367
292
|
return { description, sourceContent: '' };
|
|
368
293
|
}
|
|
369
294
|
|
|
370
|
-
/**
|
|
371
|
-
* Process a $parent source — fetch sources from the parent server without claiming.
|
|
372
|
-
*
|
|
373
|
-
* Fetches available sources via GET /newsletter-sources (no claimFor parameter),
|
|
374
|
-
* filters out any already tracked locally in `content-sources`, and returns
|
|
375
|
-
* the first available source.
|
|
376
|
-
*
|
|
377
|
-
* @param {object} assistant - BEM assistant
|
|
378
|
-
* @param {object} entry - Content entry config
|
|
379
|
-
* @param {object} admin - Firebase Admin SDK
|
|
380
|
-
* @param {object} Manager - BEM Manager instance
|
|
381
|
-
* @returns {Promise<{ id, title, url, content, summary }|null>} Selected source or null
|
|
382
|
-
*/
|
|
383
|
-
async function processParentSource(assistant, entry, admin, Manager) {
|
|
384
|
-
const parentUrl = Manager?.getParentApiUrl?.();
|
|
385
|
-
|
|
386
|
-
if (!parentUrl) {
|
|
387
|
-
assistant.log('processParentSource(): No parent URL configured');
|
|
388
|
-
return null;
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
const categories = entry.categories || [];
|
|
392
|
-
const allSources = [];
|
|
393
|
-
|
|
394
|
-
// Fetch sources from each category (or all if no categories)
|
|
395
|
-
const categoriesToFetch = categories.length ? categories : [''];
|
|
396
|
-
|
|
397
|
-
for (const category of categoriesToFetch) {
|
|
398
|
-
const query = {
|
|
399
|
-
limit: 3,
|
|
400
|
-
backendManagerKey: process.env.BACKEND_MANAGER_KEY,
|
|
401
|
-
};
|
|
402
|
-
|
|
403
|
-
if (category) {
|
|
404
|
-
query.category = category;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
const data = await fetch(`${parentUrl}/newsletter-sources`, {
|
|
408
|
-
method: 'get',
|
|
409
|
-
response: 'json',
|
|
410
|
-
timeout: 60000,
|
|
411
|
-
query: query,
|
|
412
|
-
}).catch((e) => {
|
|
413
|
-
assistant.error(`processParentSource(): Failed to fetch sources for category=${category}: ${e.message}`);
|
|
414
|
-
return null;
|
|
415
|
-
});
|
|
416
|
-
|
|
417
|
-
if (data?.sources?.length) {
|
|
418
|
-
allSources.push(...data.sources);
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
if (!allSources.length) {
|
|
423
|
-
assistant.log('processParentSource(): No sources available from parent');
|
|
424
|
-
return null;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
// Filter out already-used sources by checking content-sources collection locally
|
|
428
|
-
if (admin) {
|
|
429
|
-
const usedUrls = new Set();
|
|
430
|
-
const snapshot = await admin.firestore()
|
|
431
|
-
.collection(CONTENT_SOURCES_COLLECTION)
|
|
432
|
-
.where('origin', '==', '$parent')
|
|
433
|
-
.select('url')
|
|
434
|
-
.get()
|
|
435
|
-
.catch((e) => {
|
|
436
|
-
assistant.error('processParentSource(): Error querying tracked sources (continuing)', e);
|
|
437
|
-
return { docs: [] };
|
|
438
|
-
});
|
|
439
|
-
|
|
440
|
-
snapshot.docs.forEach((doc) => {
|
|
441
|
-
const data = doc.data();
|
|
442
|
-
if (data.url) {
|
|
443
|
-
usedUrls.add(data.url);
|
|
444
|
-
}
|
|
445
|
-
});
|
|
446
|
-
|
|
447
|
-
const available = allSources.filter((s) => !usedUrls.has(s.url || s.id));
|
|
448
|
-
|
|
449
|
-
if (!available.length) {
|
|
450
|
-
assistant.log('processParentSource(): All parent sources already used');
|
|
451
|
-
return null;
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
return available[0];
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
// No admin — just return the first source
|
|
458
|
-
return allSources[0];
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
/**
|
|
462
|
-
* Process a $feed: source — fetch, parse, select unprocessed item, extract content.
|
|
463
|
-
*
|
|
464
|
-
* @param {object} assistant - BEM assistant
|
|
465
|
-
* @param {string} feedUrl - RSS/Atom/JSON feed URL
|
|
466
|
-
* @param {string} brandId - Brand ID for tracking
|
|
467
|
-
* @param {object} admin - Firebase Admin SDK
|
|
468
|
-
* @returns {Promise<{ item: FeedItem, content: string }|null>} Selected item + extracted content, or null
|
|
469
|
-
*/
|
|
470
|
-
async function processFeedSource(assistant, feedUrl, brandId, admin) {
|
|
471
|
-
// Fetch the feed
|
|
472
|
-
const feedText = await fetch(feedUrl, {
|
|
473
|
-
timeout: 30000,
|
|
474
|
-
tries: 2,
|
|
475
|
-
response: 'text',
|
|
476
|
-
headers: { 'User-Agent': USER_AGENT },
|
|
477
|
-
}).catch((e) => e);
|
|
478
|
-
|
|
479
|
-
if (feedText instanceof Error) {
|
|
480
|
-
assistant.error(`processFeedSource(): Failed to fetch feed: ${feedUrl}`, feedText);
|
|
481
|
-
return null;
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
// Parse the feed
|
|
485
|
-
const { items } = parseFeed(feedText);
|
|
486
|
-
if (!items.length) {
|
|
487
|
-
assistant.log(`processFeedSource(): No items in feed: ${feedUrl}`);
|
|
488
|
-
return null;
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
assistant.log(`processFeedSource(): Parsed ${items.length} items from feed`);
|
|
492
|
-
|
|
493
|
-
// Get already-processed item IDs from Firestore
|
|
494
|
-
const processedIds = await getProcessedItemIds(admin, feedUrl).catch((e) => {
|
|
495
|
-
assistant.error('processFeedSource(): Error querying tracked items (continuing)', e);
|
|
496
|
-
return new Set();
|
|
497
|
-
});
|
|
498
|
-
|
|
499
|
-
// Filter to unprocessed items
|
|
500
|
-
const unprocessed = items.filter((item) => !processedIds.has(item.id) && !processedIds.has(item.url));
|
|
501
|
-
if (!unprocessed.length) {
|
|
502
|
-
assistant.log(`processFeedSource(): All ${items.length} items already processed for: ${feedUrl}`);
|
|
503
|
-
return null;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
assistant.log(`processFeedSource(): ${unprocessed.length} unprocessed items available`);
|
|
507
|
-
|
|
508
|
-
// Pick the first (newest) unprocessed item
|
|
509
|
-
const item = unprocessed[0];
|
|
510
|
-
|
|
511
|
-
// Extract full article content from the item URL
|
|
512
|
-
let content = '';
|
|
513
|
-
if (item.url) {
|
|
514
|
-
content = await extractArticleContent(item.url).catch((e) => {
|
|
515
|
-
assistant.error(`processFeedSource(): Failed to extract content from ${item.url} (using summary)`, e);
|
|
516
|
-
return '';
|
|
517
|
-
});
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
// Fall back to inline feed content or summary
|
|
521
|
-
if (!content || content.length < 100) {
|
|
522
|
-
content = item.content || item.summary || '';
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
return { item, content };
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
/**
|
|
529
|
-
* Query Firestore for already-processed feed item IDs.
|
|
530
|
-
*
|
|
531
|
-
* @param {object} admin - Firebase Admin SDK
|
|
532
|
-
* @param {string} feedUrl - The feed URL to query
|
|
533
|
-
* @returns {Promise<Set<string>>} Set of processed item IDs
|
|
534
|
-
*/
|
|
535
|
-
async function getProcessedItemIds(admin, feedUrl) {
|
|
536
|
-
if (!admin) {
|
|
537
|
-
return new Set();
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
const snapshot = await admin.firestore()
|
|
541
|
-
.collection(CONTENT_SOURCES_COLLECTION)
|
|
542
|
-
.where('feedUrl', '==', feedUrl)
|
|
543
|
-
.select('itemId', 'url')
|
|
544
|
-
.get();
|
|
545
|
-
|
|
546
|
-
const ids = new Set();
|
|
547
|
-
snapshot.docs.forEach((doc) => {
|
|
548
|
-
const data = doc.data();
|
|
549
|
-
if (data.itemId) {
|
|
550
|
-
ids.add(data.itemId);
|
|
551
|
-
}
|
|
552
|
-
if (data.url) {
|
|
553
|
-
ids.add(data.url);
|
|
554
|
-
}
|
|
555
|
-
});
|
|
556
|
-
|
|
557
|
-
return ids;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
/**
|
|
561
|
-
* Write a tracking doc for a processed content source.
|
|
562
|
-
*
|
|
563
|
-
* Unified tracker for all source types (feed, parent, brand).
|
|
564
|
-
* Used by both blog-auto-publisher and newsletter generator.
|
|
565
|
-
*
|
|
566
|
-
* @param {object} admin - Firebase Admin SDK
|
|
567
|
-
* @param {object} args
|
|
568
|
-
* @param {string} args.url - Unique identifier (article URL or source ID)
|
|
569
|
-
* @param {string} args.origin - Source type ('$feed:https://...', '$parent', '$brand')
|
|
570
|
-
* @param {string} [args.feedUrl] - The source feed URL (for feed sources)
|
|
571
|
-
* @param {string} [args.itemId] - Item ID within the feed
|
|
572
|
-
* @param {string} [args.itemTitle] - Item title
|
|
573
|
-
* @param {string} args.usedBy - Which system used it ('blog' or 'newsletter')
|
|
574
|
-
* @param {string} args.brandId - Brand that processed it
|
|
575
|
-
* @param {string} [args.postUrl] - Published blog post URL
|
|
576
|
-
* @param {string} [args.postSlug] - Published blog post slug
|
|
577
|
-
*/
|
|
578
|
-
async function trackContentSource(admin, { url, origin, feedUrl, itemId, itemTitle, usedBy, brandId, postUrl, postSlug }) {
|
|
579
|
-
const docId = contentSourceHash(origin || '', url || '');
|
|
580
|
-
const nowISO = new Date().toISOString();
|
|
581
|
-
const nowUNIX = Math.round(Date.now() / 1000);
|
|
582
|
-
|
|
583
|
-
await admin.firestore().doc(`${CONTENT_SOURCES_COLLECTION}/${docId}`).set({
|
|
584
|
-
url: url || null,
|
|
585
|
-
origin: origin || null,
|
|
586
|
-
feedUrl: feedUrl || null,
|
|
587
|
-
itemId: itemId || null,
|
|
588
|
-
itemTitle: itemTitle || null,
|
|
589
|
-
usedBy: usedBy || null,
|
|
590
|
-
brandId: brandId || null,
|
|
591
|
-
postUrl: postUrl || null,
|
|
592
|
-
postSlug: postSlug || null,
|
|
593
|
-
metadata: {
|
|
594
|
-
created: { timestamp: nowISO, timestampUNIX: nowUNIX },
|
|
595
|
-
updated: { timestamp: nowISO, timestampUNIX: nowUNIX },
|
|
596
|
-
},
|
|
597
|
-
});
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
/**
|
|
601
|
-
* Deterministic hash for a content source — used as the Firestore doc ID.
|
|
602
|
-
*/
|
|
603
|
-
function contentSourceHash(origin, url) {
|
|
604
|
-
return crypto.createHash('sha256').update(`${origin}::${url}`).digest('hex').slice(0, 20);
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
function getURLContent(url) {
|
|
608
|
-
return fetch(url, {
|
|
609
|
-
timeout: 120000,
|
|
610
|
-
tries: 3,
|
|
611
|
-
response: 'raw',
|
|
612
|
-
headers: {
|
|
613
|
-
'User-Agent': USER_AGENT,
|
|
614
|
-
},
|
|
615
|
-
})
|
|
616
|
-
.then(async (res) => {
|
|
617
|
-
const contentType = res.headers.get('content-type');
|
|
618
|
-
const text = await res.text();
|
|
619
|
-
|
|
620
|
-
return extractBodyContent(text, contentType, url);
|
|
621
|
-
});
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
function isURL(url) {
|
|
625
|
-
try {
|
|
626
|
-
return !!new URL(url);
|
|
627
|
-
} catch (e) {
|
|
628
|
-
return false;
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
function extractBodyContent(text, contentType, url) {
|
|
633
|
-
const parsed = tryParse(text);
|
|
634
|
-
|
|
635
|
-
// Try JSON
|
|
636
|
-
if (parsed) {
|
|
637
|
-
// If it's from rss.app, extract the content
|
|
638
|
-
if (parsed.items) {
|
|
639
|
-
return parsed.items.map((i) => `${i.title}: ${i.content_text}`).join('\n');
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
// If we can't recognize the JSON, return the original text
|
|
643
|
-
return text;
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
// Extract the content within the body tag
|
|
647
|
-
const bodyMatch = text.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
|
|
648
|
-
if (!bodyMatch) {
|
|
649
|
-
return '';
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
let bodyContent = bodyMatch[1];
|
|
653
|
-
|
|
654
|
-
// Remove script and meta tags
|
|
655
|
-
bodyContent = bodyContent.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
|
|
656
|
-
bodyContent = bodyContent.replace(/<meta[^>]*>/gi, '');
|
|
657
|
-
|
|
658
|
-
// Remove remaining HTML tags
|
|
659
|
-
return bodyContent.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
function tryParse(json) {
|
|
663
|
-
try {
|
|
664
|
-
return JSON5.parse(json);
|
|
665
|
-
} catch (e) {
|
|
666
|
-
return null;
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
|
|
670
295
|
function randomize(array) {
|
|
671
296
|
return array.sort(() => Math.random() - 0.5);
|
|
672
297
|
}
|
|
673
298
|
|
|
674
|
-
//
|
|
299
|
+
// Re-export shared functions for backwards compatibility (newsletter + tests import from here)
|
|
675
300
|
module.exports.resolveSource = resolveSource;
|
|
676
301
|
module.exports.processFeedSource = processFeedSource;
|
|
677
302
|
module.exports.contentSourceHash = contentSourceHash;
|
|
@@ -130,7 +130,7 @@ function blocksToPost(json) {
|
|
|
130
130
|
* @param {string} [args.postPath='ghostii'] - Sub-folder under _posts/{year}/
|
|
131
131
|
* @returns {Promise<object>} { post, url, slug, path } — `url` is the public blog URL
|
|
132
132
|
*/
|
|
133
|
-
async function publishArticle(assistant, { brand, article, id, author, postPath }) {
|
|
133
|
+
async function publishArticle(assistant, { brand, article, id, author, postPath, source }) {
|
|
134
134
|
const baseUrl = (brand.brand.url || '').replace(/^https?:\/\//, '').replace(/\/$/, '');
|
|
135
135
|
const apiUrl = `https://api.${baseUrl}`;
|
|
136
136
|
|
|
@@ -159,6 +159,7 @@ async function publishArticle(assistant, { brand, article, id, author, postPath
|
|
|
159
159
|
categories: article.categories,
|
|
160
160
|
tags: article.keywords,
|
|
161
161
|
postPath: postPath || 'ghostii',
|
|
162
|
+
source: source || null,
|
|
162
163
|
githubUser: brand.github.user,
|
|
163
164
|
githubRepo: brand.github.repo,
|
|
164
165
|
},
|