@rmdes/indiekit-endpoint-microsub 1.0.63 → 1.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cache/redis.js +0 -31
- package/lib/controllers/block.js +0 -2
- package/lib/controllers/channels.js +0 -2
- package/lib/controllers/events.js +0 -2
- package/lib/controllers/follow.js +0 -2
- package/lib/controllers/mute.js +0 -2
- package/lib/controllers/preview.js +0 -2
- package/lib/controllers/reader/channel.js +0 -1
- package/lib/controllers/reader/index.js +2 -38
- package/lib/controllers/search.js +0 -2
- package/lib/controllers/timeline.js +0 -2
- package/lib/feeds/discovery.js +1 -1
- package/lib/feeds/normalizer.js +2 -2
- package/lib/media/proxy.js +5 -5
- package/lib/polling/scheduler.js +4 -14
- package/lib/polling/tier.js +6 -51
- package/lib/realtime/broker.js +3 -75
- package/lib/storage/channels.js +1 -1
- package/lib/storage/feeds.js +1 -17
- package/lib/storage/filters.js +26 -197
- package/lib/storage/items-read-state.js +30 -59
- package/lib/storage/items-retention.js +1 -3
- package/lib/storage/items.js +0 -14
- package/lib/utils/blogroll-notify.js +3 -3
- package/lib/utils/constants.js +7 -0
- package/lib/utils/jf2.js +0 -109
- package/lib/utils/pagination.js +4 -11
- package/lib/utils/sanitize.js +1 -2
- package/lib/utils/validation.js +0 -10
- package/lib/webmention/processor.js +2 -95
- package/lib/websub/handler.js +2 -2
- package/lib/websub/subscriber.js +0 -17
- package/locales/en.json +3 -27
- package/package.json +1 -4
- package/lib/search/indexer.js +0 -90
- package/lib/storage/items-search.js +0 -34
- package/lib/storage/read-state.js +0 -109
- package/lib/websub/discovery.js +0 -129
package/lib/cache/redis.js
CHANGED
|
@@ -119,24 +119,6 @@ export async function setCache(redis, key, value, ttl = 300) {
|
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
/**
|
|
123
|
-
* Delete value from cache
|
|
124
|
-
* @param {object} redis - Redis client
|
|
125
|
-
* @param {string} key - Cache key
|
|
126
|
-
* @returns {Promise<void>}
|
|
127
|
-
*/
|
|
128
|
-
export async function deleteCache(redis, key) {
|
|
129
|
-
if (!redis) {
|
|
130
|
-
return;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
try {
|
|
134
|
-
await redis.del(nsKey(key));
|
|
135
|
-
} catch {
|
|
136
|
-
// Ignore cache errors
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
122
|
/**
|
|
141
123
|
* Publish event to channel
|
|
142
124
|
* @param {object} redis - Redis client
|
|
@@ -185,16 +167,3 @@ export async function subscribeToChannel(redis, channel, callback) {
|
|
|
185
167
|
}
|
|
186
168
|
}
|
|
187
169
|
|
|
188
|
-
/**
|
|
189
|
-
* Cleanup Redis connection on shutdown
|
|
190
|
-
*/
|
|
191
|
-
export async function closeRedis() {
|
|
192
|
-
if (redisClient) {
|
|
193
|
-
try {
|
|
194
|
-
await redisClient.quit();
|
|
195
|
-
redisClient = undefined;
|
|
196
|
-
} catch {
|
|
197
|
-
// Ignore cleanup errors
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
package/lib/controllers/block.js
CHANGED
package/lib/controllers/mute.js
CHANGED
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
updateChannelSettings,
|
|
11
11
|
deleteChannel,
|
|
12
12
|
} from "../../storage/channels.js";
|
|
13
|
-
import { getFeedsForChannel } from "../../storage/feeds.js";
|
|
14
13
|
import { getTimelineItems } from "../../storage/items.js";
|
|
15
14
|
import { countReadItems } from "../../storage/items-read-state.js";
|
|
16
15
|
import { getUserId } from "../../utils/auth.js";
|
|
@@ -1,45 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Reader controller
|
|
2
|
+
* Reader controller aggregator. Bundles handlers from sibling modules into the
|
|
3
|
+
* `readerController` object consumed by route wiring in the plugin root.
|
|
3
4
|
* @module controllers/reader
|
|
4
5
|
*/
|
|
5
6
|
|
|
6
|
-
export {
|
|
7
|
-
index,
|
|
8
|
-
channels,
|
|
9
|
-
newChannel,
|
|
10
|
-
createChannelAction,
|
|
11
|
-
channel,
|
|
12
|
-
channelHtml,
|
|
13
|
-
settings,
|
|
14
|
-
updateSettings,
|
|
15
|
-
deleteChannelAction,
|
|
16
|
-
} from "./channel.js";
|
|
17
|
-
|
|
18
|
-
export {
|
|
19
|
-
feeds,
|
|
20
|
-
addFeed,
|
|
21
|
-
removeFeed,
|
|
22
|
-
feedDetails,
|
|
23
|
-
editFeedForm,
|
|
24
|
-
updateFeedUrl,
|
|
25
|
-
rediscoverFeed,
|
|
26
|
-
refreshFeed,
|
|
27
|
-
} from "./feed.js";
|
|
28
|
-
|
|
29
|
-
export {
|
|
30
|
-
timeline,
|
|
31
|
-
timelineHtml,
|
|
32
|
-
markAllRead,
|
|
33
|
-
markViewRead,
|
|
34
|
-
item,
|
|
35
|
-
} from "./timeline.js";
|
|
36
|
-
|
|
37
|
-
export { compose, submitCompose } from "./compose.js";
|
|
38
|
-
|
|
39
|
-
export { searchPage, searchFeeds, subscribe } from "./search.js";
|
|
40
|
-
|
|
41
|
-
export { deck, deckSettings, saveDeckSettings } from "./deck.js";
|
|
42
|
-
|
|
43
7
|
import {
|
|
44
8
|
index,
|
|
45
9
|
channels,
|
package/lib/feeds/discovery.js
CHANGED
|
@@ -78,7 +78,7 @@ export async function discoverAndValidateFeeds(url) {
|
|
|
78
78
|
* @param {Array} feeds - Array of feed objects
|
|
79
79
|
* @returns {Array} Filtered array of main content feeds
|
|
80
80
|
*/
|
|
81
|
-
|
|
81
|
+
function filterMainFeeds(feeds) {
|
|
82
82
|
return feeds.filter((feed) => feed.valid && !feed.isCommentsFeed);
|
|
83
83
|
}
|
|
84
84
|
|
package/lib/feeds/normalizer.js
CHANGED
|
@@ -30,7 +30,7 @@ export function generateItemUid(feedUrl, itemId) {
|
|
|
30
30
|
* @param {string|Date} dateInput - Date string or Date object
|
|
31
31
|
* @returns {Date|undefined} Parsed Date or undefined if invalid
|
|
32
32
|
*/
|
|
33
|
-
|
|
33
|
+
function parseDate(dateInput) {
|
|
34
34
|
if (!dateInput) {
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
@@ -108,7 +108,7 @@ export function extractPhotoUrl(photo) {
|
|
|
108
108
|
* @param {object|string} value - URL string or object with url/value property
|
|
109
109
|
* @returns {string|undefined} URL string
|
|
110
110
|
*/
|
|
111
|
-
|
|
111
|
+
function extractUrl(value) {
|
|
112
112
|
if (!value) {
|
|
113
113
|
return;
|
|
114
114
|
}
|
package/lib/media/proxy.js
CHANGED
|
@@ -98,7 +98,7 @@ const ALLOWED_TYPES = new Set([
|
|
|
98
98
|
* @param {string} url - Original image URL
|
|
99
99
|
* @returns {string} URL-safe hash
|
|
100
100
|
*/
|
|
101
|
-
|
|
101
|
+
function hashUrl(url) {
|
|
102
102
|
return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
103
103
|
}
|
|
104
104
|
|
|
@@ -107,7 +107,7 @@ export function hashUrl(url) {
|
|
|
107
107
|
* @param {string} url - Original image URL
|
|
108
108
|
* @returns {string} HMAC hex signature (16 chars)
|
|
109
109
|
*/
|
|
110
|
-
|
|
110
|
+
function signProxyUrl(url) {
|
|
111
111
|
const secret = process.env.SECRET || "microsub-default-secret";
|
|
112
112
|
return crypto
|
|
113
113
|
.createHmac("sha256", secret)
|
|
@@ -122,7 +122,7 @@ export function signProxyUrl(url) {
|
|
|
122
122
|
* @param {string} sig - Submitted signature
|
|
123
123
|
* @returns {boolean} Whether signature is valid
|
|
124
124
|
*/
|
|
125
|
-
|
|
125
|
+
function verifyProxySignature(url, sig) {
|
|
126
126
|
if (!sig) return false;
|
|
127
127
|
const expected = signProxyUrl(url);
|
|
128
128
|
// Constant-time comparison
|
|
@@ -136,7 +136,7 @@ export function verifyProxySignature(url, sig) {
|
|
|
136
136
|
* @param {string} originalUrl - Original image URL
|
|
137
137
|
* @returns {string} Proxied URL
|
|
138
138
|
*/
|
|
139
|
-
|
|
139
|
+
function getProxiedUrl(baseUrl, originalUrl) {
|
|
140
140
|
if (!originalUrl || !baseUrl) {
|
|
141
141
|
return originalUrl;
|
|
142
142
|
}
|
|
@@ -203,7 +203,7 @@ export function proxyItemImages(item, baseUrl) {
|
|
|
203
203
|
* @param {string} url - Image URL to fetch
|
|
204
204
|
* @returns {Promise<object|null>} Cached image data or null
|
|
205
205
|
*/
|
|
206
|
-
|
|
206
|
+
async function fetchImage(redis, url) {
|
|
207
207
|
// Block private/internal URLs (defense-in-depth)
|
|
208
208
|
if (await isPrivateUrl(url)) {
|
|
209
209
|
console.error(`[Microsub] Media proxy blocked private URL: ${url}`);
|
package/lib/polling/scheduler.js
CHANGED
|
@@ -35,7 +35,7 @@ export function startScheduler(indiekit) {
|
|
|
35
35
|
// Run immediately on start
|
|
36
36
|
runSchedulerCycle();
|
|
37
37
|
|
|
38
|
-
console.
|
|
38
|
+
console.info("[Microsub] Feed polling scheduler started");
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -47,7 +47,7 @@ export function stopScheduler() {
|
|
|
47
47
|
schedulerInterval = undefined;
|
|
48
48
|
}
|
|
49
49
|
indiekitInstance = undefined;
|
|
50
|
-
console.
|
|
50
|
+
console.info("[Microsub] Feed polling scheduler stopped");
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
/**
|
|
@@ -74,13 +74,13 @@ async function runSchedulerCycle() {
|
|
|
74
74
|
return;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
console.
|
|
77
|
+
console.info(`[Microsub] Processing ${feeds.length} feeds due for refresh`);
|
|
78
78
|
|
|
79
79
|
const result = await processFeedBatch(application, feeds, {
|
|
80
80
|
concurrency: BATCH_CONCURRENCY,
|
|
81
81
|
});
|
|
82
82
|
|
|
83
|
-
console.
|
|
83
|
+
console.info(
|
|
84
84
|
`[Microsub] Processed ${result.total} feeds: ${result.successful} successful, ` +
|
|
85
85
|
`${result.failed} failed, ${result.itemsAdded} new items`,
|
|
86
86
|
);
|
|
@@ -118,13 +118,3 @@ export async function refreshFeedNow(application, feedId) {
|
|
|
118
118
|
return processFeed(application, feed);
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
/**
|
|
122
|
-
* Get scheduler status
|
|
123
|
-
* @returns {object} Scheduler status
|
|
124
|
-
*/
|
|
125
|
-
export function getSchedulerStatus() {
|
|
126
|
-
return {
|
|
127
|
-
running: !!schedulerInterval,
|
|
128
|
-
processing: isRunning,
|
|
129
|
-
};
|
|
130
|
-
}
|
package/lib/polling/tier.js
CHANGED
|
@@ -23,22 +23,24 @@ const MAX_TIER = 10;
|
|
|
23
23
|
const DEFAULT_TIER = 1;
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
|
-
* Get polling interval for a tier in milliseconds
|
|
26
|
+
* Get polling interval for a tier in milliseconds.
|
|
27
|
+
* Internal helper for getNextFetchTime.
|
|
27
28
|
* @param {number} tier - Polling tier (0-10)
|
|
28
29
|
* @returns {number} Interval in milliseconds
|
|
29
30
|
*/
|
|
30
|
-
|
|
31
|
+
function getIntervalForTier(tier) {
|
|
31
32
|
const clampedTier = Math.max(MIN_TIER, Math.min(MAX_TIER, tier));
|
|
32
33
|
const minutes = Math.pow(2, clampedTier);
|
|
33
34
|
return minutes * 60 * 1000;
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
|
-
* Get next fetch time based on tier
|
|
38
|
+
* Get next fetch time based on tier.
|
|
39
|
+
* Internal helper for calculateNewTier.
|
|
38
40
|
* @param {number} tier - Polling tier
|
|
39
41
|
* @returns {Date} Next fetch time
|
|
40
42
|
*/
|
|
41
|
-
|
|
43
|
+
function getNextFetchTime(tier) {
|
|
42
44
|
const interval = getIntervalForTier(tier);
|
|
43
45
|
return new Date(Date.now() + interval);
|
|
44
46
|
}
|
|
@@ -90,50 +92,3 @@ export function calculateNewTier(options) {
|
|
|
90
92
|
};
|
|
91
93
|
}
|
|
92
94
|
|
|
93
|
-
/**
|
|
94
|
-
* Get initial tier for a new feed subscription
|
|
95
|
-
* @returns {object} Initial tier settings
|
|
96
|
-
*/
|
|
97
|
-
export function getInitialTier() {
|
|
98
|
-
return {
|
|
99
|
-
tier: MIN_TIER, // Start at tier 0 for immediate first fetch
|
|
100
|
-
consecutiveUnchanged: 0,
|
|
101
|
-
nextFetchAt: new Date(), // Fetch immediately
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Determine if a feed is due for fetching
|
|
107
|
-
* @param {object} feed - Feed document
|
|
108
|
-
* @returns {boolean} Whether the feed should be fetched
|
|
109
|
-
*/
|
|
110
|
-
export function isDueForFetch(feed) {
|
|
111
|
-
if (!feed.nextFetchAt) {
|
|
112
|
-
return true;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return new Date(feed.nextFetchAt) <= new Date();
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Get human-readable description of polling interval
|
|
120
|
-
* @param {number} tier - Polling tier
|
|
121
|
-
* @returns {string} Description
|
|
122
|
-
*/
|
|
123
|
-
export function getTierDescription(tier) {
|
|
124
|
-
const minutes = Math.pow(2, tier);
|
|
125
|
-
|
|
126
|
-
if (minutes < 60) {
|
|
127
|
-
return `every ${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
const hours = minutes / 60;
|
|
131
|
-
if (hours < 24) {
|
|
132
|
-
return `every ${hours.toFixed(1)} hour${hours === 1 ? "" : "s"}`;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const days = hours / 24;
|
|
136
|
-
return `every ${days.toFixed(1)} day${days === 1 ? "" : "s"}`;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
export { MIN_TIER, MAX_TIER, DEFAULT_TIER };
|
package/lib/realtime/broker.js
CHANGED
|
@@ -84,18 +84,6 @@ export function subscribeClient(response, channelId) {
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
/**
|
|
88
|
-
* Unsubscribe a client from a channel
|
|
89
|
-
* @param {object} response - Express response object
|
|
90
|
-
* @param {string} channelId - Channel ID
|
|
91
|
-
*/
|
|
92
|
-
export function unsubscribeClient(response, channelId) {
|
|
93
|
-
const client = clients.get(response);
|
|
94
|
-
if (client) {
|
|
95
|
-
client.channels.delete(channelId);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
87
|
/**
|
|
100
88
|
* Send an event to a specific client
|
|
101
89
|
* @param {object} response - Express response object
|
|
@@ -113,26 +101,13 @@ export function sendEvent(response, event, data) {
|
|
|
113
101
|
}
|
|
114
102
|
|
|
115
103
|
/**
|
|
116
|
-
* Broadcast an event to all clients
|
|
117
|
-
*
|
|
118
|
-
* @param {string} event - Event name
|
|
119
|
-
* @param {object} data - Event data
|
|
120
|
-
*/
|
|
121
|
-
export function broadcastToChannel(channelId, event, data) {
|
|
122
|
-
for (const client of clients.values()) {
|
|
123
|
-
if (client.channels.has(channelId)) {
|
|
124
|
-
sendEvent(client.response, event, data);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Broadcast an event to all clients for a user
|
|
104
|
+
* Broadcast an event to all clients for a user.
|
|
105
|
+
* Internal helper — only invoked by the Redis fan-out in handleRedisEvent.
|
|
131
106
|
* @param {string} userId - User ID
|
|
132
107
|
* @param {string} event - Event name
|
|
133
108
|
* @param {object} data - Event data
|
|
134
109
|
*/
|
|
135
|
-
|
|
110
|
+
function broadcastToUser(userId, event, data) {
|
|
136
111
|
for (const client of clients.values()) {
|
|
137
112
|
if (client.userId === userId) {
|
|
138
113
|
sendEvent(client.response, event, data);
|
|
@@ -140,17 +115,6 @@ export function broadcastToUser(userId, event, data) {
|
|
|
140
115
|
}
|
|
141
116
|
}
|
|
142
117
|
|
|
143
|
-
/**
|
|
144
|
-
* Broadcast an event to all connected clients
|
|
145
|
-
* @param {string} event - Event name
|
|
146
|
-
* @param {object} data - Event data
|
|
147
|
-
*/
|
|
148
|
-
export function broadcastToAll(event, data) {
|
|
149
|
-
for (const client of clients.values()) {
|
|
150
|
-
sendEvent(client.response, event, data);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
|
|
154
118
|
/**
|
|
155
119
|
* Set up Redis subscription for a user
|
|
156
120
|
* @param {string} userId - User ID
|
|
@@ -208,39 +172,3 @@ function handleRedisEvent(userId, data) {
|
|
|
208
172
|
}
|
|
209
173
|
}
|
|
210
174
|
|
|
211
|
-
/**
|
|
212
|
-
* Get broker statistics
|
|
213
|
-
* @returns {object} Statistics
|
|
214
|
-
*/
|
|
215
|
-
export function getStats() {
|
|
216
|
-
const userCounts = new Map();
|
|
217
|
-
for (const client of clients.values()) {
|
|
218
|
-
const count = userCounts.get(client.userId) || 0;
|
|
219
|
-
userCounts.set(client.userId, count + 1);
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
return {
|
|
223
|
-
totalClients: clients.size,
|
|
224
|
-
uniqueUsers: userCounts.size,
|
|
225
|
-
userSubscribers: userSubscribers.size,
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
* Clean up all connections
|
|
231
|
-
*/
|
|
232
|
-
export function cleanup() {
|
|
233
|
-
for (const client of clients.values()) {
|
|
234
|
-
clearInterval(client.pingInterval);
|
|
235
|
-
}
|
|
236
|
-
clients.clear();
|
|
237
|
-
|
|
238
|
-
for (const subscriber of userSubscribers.values()) {
|
|
239
|
-
try {
|
|
240
|
-
subscriber.quit();
|
|
241
|
-
} catch {
|
|
242
|
-
// Ignore cleanup errors
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
userSubscribers.clear();
|
|
246
|
-
}
|
package/lib/storage/channels.js
CHANGED
|
@@ -29,7 +29,7 @@ const CHANNEL_COLORS = [
|
|
|
29
29
|
* @param {number} order - Channel order index
|
|
30
30
|
* @returns {string} Hex color
|
|
31
31
|
*/
|
|
32
|
-
|
|
32
|
+
function getChannelColor(order) {
|
|
33
33
|
return CHANNEL_COLORS[Math.abs(order) % CHANNEL_COLORS.length];
|
|
34
34
|
}
|
|
35
35
|
|
package/lib/storage/feeds.js
CHANGED
|
@@ -45,7 +45,7 @@ export function normalizeUrl(url) {
|
|
|
45
45
|
* @param {string} url - Feed URL to check
|
|
46
46
|
* @returns {Promise<object|null>} Existing feed with channel info, or null
|
|
47
47
|
*/
|
|
48
|
-
|
|
48
|
+
async function findFeedAcrossChannels(application, url) {
|
|
49
49
|
const collection = getCollection(application);
|
|
50
50
|
const normalized = normalizeUrl(url);
|
|
51
51
|
|
|
@@ -427,19 +427,3 @@ export async function updateFeedStatus(application, id, status) {
|
|
|
427
427
|
});
|
|
428
428
|
}
|
|
429
429
|
|
|
430
|
-
/**
|
|
431
|
-
* Get feeds with errors
|
|
432
|
-
* @param {object} application - Indiekit application
|
|
433
|
-
* @param {number} [minErrors=3] - Minimum consecutive errors
|
|
434
|
-
* @returns {Promise<Array>} Array of feeds with errors
|
|
435
|
-
*/
|
|
436
|
-
export async function getFeedsWithErrors(application, minErrors = 3) {
|
|
437
|
-
const collection = getCollection(application);
|
|
438
|
-
|
|
439
|
-
return collection
|
|
440
|
-
.find({
|
|
441
|
-
status: "error",
|
|
442
|
-
consecutiveErrors: { $gte: minErrors },
|
|
443
|
-
})
|
|
444
|
-
.toArray();
|
|
445
|
-
}
|