@rmdes/indiekit-endpoint-microsub 1.0.66 → 1.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/feeds/normalizer-rss.js +13 -0
- package/lib/feeds/source-meta.js +46 -0
- package/lib/polling/processor.js +17 -0
- package/package.json +1 -1
|
@@ -62,6 +62,19 @@ export function normalizeItem(item, feedUrl, feedType) {
|
|
|
62
62
|
};
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
// Item-level <source> (RSS 2.0) — per-item origin in aggregator feeds
|
|
66
|
+
// (e.g. a community feed where every item comes from a different user's
|
|
67
|
+
// feed). Feedparser exposes it as { url, title }.
|
|
68
|
+
if (item.source && (item.source.url || item.source.title)) {
|
|
69
|
+
normalized._source.itemSource = {
|
|
70
|
+
url: item.source.url,
|
|
71
|
+
title: item.source.title,
|
|
72
|
+
};
|
|
73
|
+
if (!normalized.author && item.source.title) {
|
|
74
|
+
normalized.author = { type: "card", name: item.source.title };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
65
78
|
// Categories/tags
|
|
66
79
|
if (item.categories && item.categories.length > 0) {
|
|
67
80
|
normalized.category = item.categories;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve metadata (name, photo, site url) of an item-level <source> feed.
|
|
3
|
+
* Aggregator feeds attribute each item to its originating feed via the
|
|
4
|
+
* RSS 2.0 <source url="..."> element; fetching that feed's channel image
|
|
5
|
+
* gives us the author's avatar. Results are cached in memory so each
|
|
6
|
+
* distinct source feed is fetched at most once per TTL.
|
|
7
|
+
* @module feeds/source-meta
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { fetchAndParseFeed } from "./fetcher.js";
|
|
11
|
+
|
|
12
|
+
const TTL = 24 * 60 * 60 * 1000; // 24h — feed titles/avatars change rarely
|
|
13
|
+
const MAX_CACHE_ENTRIES = 500;
|
|
14
|
+
|
|
15
|
+
/** @type {Map<string, { promise: Promise<object|undefined>, fetchedAt: number }>} */
|
|
16
|
+
const cache = new Map();
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Resolve a source feed's metadata, cached
|
|
20
|
+
* @param {string} url - Source feed URL
|
|
21
|
+
* @returns {Promise<object|undefined>} { name, photo, url } or undefined on failure
|
|
22
|
+
*/
|
|
23
|
+
export async function resolveSourceFeedMeta(url) {
|
|
24
|
+
const now = Date.now();
|
|
25
|
+
const hit = cache.get(url);
|
|
26
|
+
if (hit && now - hit.fetchedAt < TTL) {
|
|
27
|
+
return hit.promise;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// ponytail: crude full-clear eviction; LRU if churn ever matters
|
|
31
|
+
if (cache.size >= MAX_CACHE_ENTRIES) {
|
|
32
|
+
cache.clear();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const promise = fetchAndParseFeed(url, { timeout: 10_000 })
|
|
36
|
+
.then((parsed) => ({
|
|
37
|
+
name: parsed.name,
|
|
38
|
+
photo: parsed.photo,
|
|
39
|
+
url: parsed.url,
|
|
40
|
+
}))
|
|
41
|
+
// Failures are cached too (negative cache) — avoids hammering dead feeds
|
|
42
|
+
.catch(() => {});
|
|
43
|
+
|
|
44
|
+
cache.set(url, { promise, fetchedAt: now });
|
|
45
|
+
return promise;
|
|
46
|
+
}
|
package/lib/polling/processor.js
CHANGED
|
@@ -9,6 +9,7 @@ const MAX_ITEMS_PER_CYCLE = 100; // Max items to process per feed per cycle
|
|
|
9
9
|
import { getRedisClient, publishEvent } from "../cache/redis.js";
|
|
10
10
|
import { detectCapabilities } from "../feeds/capabilities.js";
|
|
11
11
|
import { fetchAndParseFeed } from "../feeds/fetcher.js";
|
|
12
|
+
import { resolveSourceFeedMeta } from "../feeds/source-meta.js";
|
|
12
13
|
import { getChannelById } from "../storage/channels.js";
|
|
13
14
|
import {
|
|
14
15
|
updateFeed,
|
|
@@ -100,6 +101,22 @@ export async function processFeed(application, feed) {
|
|
|
100
101
|
item._source.source_type = classifyUrl(feed.url).type;
|
|
101
102
|
}
|
|
102
103
|
|
|
104
|
+
// Aggregator feeds: item-level <source> names the originating feed.
|
|
105
|
+
// Resolve its channel image/homepage (cached, one fetch per source
|
|
106
|
+
// feed per day) so the card shows the actual author's avatar.
|
|
107
|
+
const itemSource = item._source.itemSource;
|
|
108
|
+
if (itemSource?.url && !item.author?.photo) {
|
|
109
|
+
const meta = await resolveSourceFeedMeta(itemSource.url);
|
|
110
|
+
if (meta) {
|
|
111
|
+
item.author = {
|
|
112
|
+
type: "card",
|
|
113
|
+
name: item.author?.name || itemSource.title || meta.name,
|
|
114
|
+
url: item.author?.url || meta.url,
|
|
115
|
+
photo: meta.photo,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
103
120
|
// Store the item
|
|
104
121
|
const stored = await addItem(application, {
|
|
105
122
|
channelId: feed.channelId,
|
package/package.json
CHANGED