@anyul/koishi-plugin-rss 5.2.3 → 5.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -37
- package/lib/commands/error-handler.js +13 -1
- package/lib/commands/index.d.ts +3 -0
- package/lib/commands/index.js +7 -1
- package/lib/commands/runtime.d.ts +17 -0
- package/lib/commands/runtime.js +27 -0
- package/lib/commands/subscription-create.d.ts +23 -0
- package/lib/commands/subscription-create.js +145 -0
- package/lib/commands/web-monitor.d.ts +15 -0
- package/lib/commands/web-monitor.js +222 -0
- package/lib/config.js +7 -1
- package/lib/constants.d.ts +1 -1
- package/lib/constants.js +46 -83
- package/lib/core/ai-cache.d.ts +27 -0
- package/lib/core/ai-cache.js +169 -0
- package/lib/core/ai-client.d.ts +12 -0
- package/lib/core/ai-client.js +65 -0
- package/lib/core/ai-selector.d.ts +2 -0
- package/lib/core/ai-selector.js +80 -0
- package/lib/core/ai-summary.d.ts +10 -0
- package/lib/core/ai-summary.js +73 -0
- package/lib/core/ai-utils.d.ts +10 -0
- package/lib/core/ai-utils.js +104 -0
- package/lib/core/ai.d.ts +3 -91
- package/lib/core/ai.js +13 -522
- package/lib/core/feeder-arg.d.ts +17 -0
- package/lib/core/feeder-arg.js +234 -0
- package/lib/core/feeder-runtime.d.ts +96 -0
- package/lib/core/feeder-runtime.js +233 -0
- package/lib/core/feeder.d.ts +3 -5
- package/lib/core/feeder.js +61 -358
- package/lib/core/item-processor-runtime.d.ts +46 -0
- package/lib/core/item-processor-runtime.js +215 -0
- package/lib/core/item-processor-template.d.ts +16 -0
- package/lib/core/item-processor-template.js +158 -0
- package/lib/core/item-processor.d.ts +1 -15
- package/lib/core/item-processor.js +44 -319
- package/lib/core/notification-queue-retry.d.ts +25 -0
- package/lib/core/notification-queue-retry.js +78 -0
- package/lib/core/notification-queue-sender.d.ts +20 -0
- package/lib/core/notification-queue-sender.js +118 -0
- package/lib/core/notification-queue-store.d.ts +19 -0
- package/lib/core/notification-queue-store.js +137 -0
- package/lib/core/notification-queue-types.d.ts +49 -0
- package/lib/core/notification-queue-types.js +2 -0
- package/lib/core/notification-queue.d.ts +11 -72
- package/lib/core/notification-queue.js +81 -258
- package/lib/core/search-format.d.ts +3 -0
- package/lib/core/search-format.js +36 -0
- package/lib/core/search-providers.d.ts +13 -0
- package/lib/core/search-providers.js +175 -0
- package/lib/core/search-rotation.d.ts +4 -0
- package/lib/core/search-rotation.js +55 -0
- package/lib/core/search-service.d.ts +3 -0
- package/lib/core/search-service.js +100 -0
- package/lib/core/search-types.d.ts +39 -0
- package/lib/core/search-types.js +2 -0
- package/lib/core/search.d.ts +4 -101
- package/lib/core/search.js +10 -508
- package/lib/index.js +27 -381
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/lib/types.d.ts +27 -6
- package/lib/utils/legacy-config.d.ts +12 -0
- package/lib/utils/legacy-config.js +56 -0
- package/lib/utils/logger.js +50 -29
- package/lib/utils/proxy.d.ts +3 -0
- package/lib/utils/proxy.js +14 -0
- package/lib/utils/structured-logger.d.ts +7 -3
- package/lib/utils/structured-logger.js +26 -19
- package/package.json +1 -1
package/lib/core/feeder.js
CHANGED
|
@@ -33,331 +33,50 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.findRssItem =
|
|
37
|
-
exports.getLastContent = getLastContent;
|
|
38
|
-
exports.formatArg = formatArg;
|
|
39
|
-
exports.mixinArg = mixinArg;
|
|
36
|
+
exports.getLastContent = exports.findRssItem = exports.mixinArg = exports.formatArg = void 0;
|
|
40
37
|
exports.feeder = feeder;
|
|
41
38
|
exports.startFeeder = startFeeder;
|
|
42
39
|
exports.stopFeeder = stopFeeder;
|
|
43
40
|
const koishi_1 = require("koishi");
|
|
41
|
+
const common_1 = require("../utils/common");
|
|
44
42
|
const error_handler_1 = require("../utils/error-handler");
|
|
45
43
|
const error_tracker_1 = require("../utils/error-tracker");
|
|
44
|
+
const legacy_config_1 = require("../utils/legacy-config");
|
|
46
45
|
const logger_1 = require("../utils/logger");
|
|
47
|
-
const
|
|
48
|
-
const
|
|
49
|
-
const
|
|
46
|
+
const feeder_arg_1 = require("./feeder-arg");
|
|
47
|
+
const feeder_runtime_1 = require("./feeder-runtime");
|
|
48
|
+
const notification_queue_retry_1 = require("./notification-queue-retry");
|
|
49
|
+
var feeder_arg_2 = require("./feeder-arg");
|
|
50
|
+
Object.defineProperty(exports, "formatArg", { enumerable: true, get: function () { return feeder_arg_2.formatArg; } });
|
|
51
|
+
Object.defineProperty(exports, "mixinArg", { enumerable: true, get: function () { return feeder_arg_2.mixinArg; } });
|
|
52
|
+
var feeder_runtime_2 = require("./feeder-runtime");
|
|
53
|
+
Object.defineProperty(exports, "findRssItem", { enumerable: true, get: function () { return feeder_runtime_2.findRssItem; } });
|
|
54
|
+
Object.defineProperty(exports, "getLastContent", { enumerable: true, get: function () { return feeder_runtime_2.getLastContent; } });
|
|
50
55
|
let interval = null;
|
|
51
56
|
let queueInterval = null;
|
|
52
|
-
function
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
function createFeedDebug(config, rssItem) {
|
|
63
|
-
return (0, logger_1.createDebugWithContext)(config, buildFeedLogContext(rssItem));
|
|
64
|
-
}
|
|
65
|
-
function findRssItem(rssList, keyword) {
|
|
66
|
-
// 优先匹配列表索引(用户看到的序号 1, 2, 3...)
|
|
67
|
-
if (typeof keyword === 'number' || /^\d+$/.test(String(keyword))) {
|
|
68
|
-
const listIndex = parseInt(String(keyword)) - 1; // 转换为数组索引(0-based)
|
|
69
|
-
if (listIndex >= 0 && listIndex < rssList.length) {
|
|
70
|
-
return rssList[listIndex];
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
// 其他匹配方式:按 rssId、url、title 等
|
|
74
|
-
const index = ((rssList.findIndex(i => i.rssId === +keyword) + 1) ||
|
|
75
|
-
(rssList.findIndex(i => i.url == keyword) + 1) ||
|
|
76
|
-
(rssList.findIndex(i => i.url.indexOf(keyword) + 1) + 1) ||
|
|
77
|
-
(rssList.findIndex(i => i.title.indexOf(keyword) + 1) + 1)) - 1;
|
|
78
|
-
// 边界检查:确保索引有效
|
|
79
|
-
if (index < 0 || index >= rssList.length) {
|
|
80
|
-
return undefined;
|
|
81
|
-
}
|
|
82
|
-
return rssList[index];
|
|
83
|
-
}
|
|
84
|
-
function getLastContent(item, config) {
|
|
85
|
-
let arr = ['title', 'description', 'link', 'guid'];
|
|
86
|
-
let obj = Object.assign({}, ...arr.map(i => (0, koishi_1.clone)(item?.[i]) ? { [i]: item[i] } : {}));
|
|
87
|
-
return { ...obj, description: String(obj?.description).replaceAll(/\s/g, '') };
|
|
88
|
-
}
|
|
89
|
-
function formatArg(options, config) {
|
|
90
|
-
let { arg, template, auth } = options;
|
|
91
|
-
const parseArrayArg = (value) => {
|
|
92
|
-
return value
|
|
93
|
-
.split('/')
|
|
94
|
-
.map(item => item.trim())
|
|
95
|
-
.filter(Boolean);
|
|
96
|
-
};
|
|
97
|
-
// 特殊处理:提取完整的 proxyAgent URL
|
|
98
|
-
let proxyAgentUrl;
|
|
99
|
-
if (arg && arg.includes('proxyAgent:')) {
|
|
100
|
-
const match = arg.match(/proxyAgent:([^,]+)/);
|
|
101
|
-
if (match) {
|
|
102
|
-
proxyAgentUrl = match[1];
|
|
103
|
-
// 从 arg 中移除 proxyAgent,避免被 split(":") 破坏
|
|
104
|
-
arg = arg.replace(/proxyAgent:[^,]+/, '').replace(/^,|,$/g, '').replace(/,,/g, ',');
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
let json = Object.assign({}, ...(arg?.split(',')?.map((i) => ({ [i.split(":")[0]]: i.split(":")[1] })) || []));
|
|
108
|
-
let key = ["forceLength", "reverse", "timeout", "interval", "merge", "maxRssItem", "firstLoad", "bodyWidth", "bodyPadding", "filter", "block"];
|
|
109
|
-
let booleanKey = ['firstLoad', "reverse", 'merge'];
|
|
110
|
-
let numberKey = ['forceLength', "timeout", 'interval', 'maxRssItem', 'bodyWidth', 'bodyPadding'];
|
|
111
|
-
let falseContent = ['false', 'null', ''];
|
|
112
|
-
json = Object.assign({}, ...Object.keys(json).filter((i) => key.some((key) => key == i)).map((key) => ({ [key]: booleanKey.some((bkey) => bkey == key) ? !falseContent.some((c) => c == json[key]) : numberKey.some((nkey) => nkey == key) ? (+json[key]) : json[key] })));
|
|
113
|
-
if (template && config.template) {
|
|
114
|
-
json['template'] = template;
|
|
115
|
-
}
|
|
116
|
-
// Date/Number conversions
|
|
117
|
-
if (json.interval)
|
|
118
|
-
json.interval = parseInt(json.interval) * 1000;
|
|
119
|
-
if (json.forceLength)
|
|
120
|
-
json.forceLength = parseInt(json.forceLength);
|
|
121
|
-
// Array conversions
|
|
122
|
-
if (json.filter && typeof json.filter === 'string')
|
|
123
|
-
json.filter = parseArrayArg(json.filter);
|
|
124
|
-
if (json.block && typeof json.block === 'string')
|
|
125
|
-
json.block = parseArrayArg(json.block);
|
|
126
|
-
// Proxy Argument Parsing (使用提取的完整 URL)
|
|
127
|
-
if (proxyAgentUrl) {
|
|
128
|
-
if (['false', 'none', ''].includes(String(proxyAgentUrl))) {
|
|
129
|
-
json.proxyAgent = { enabled: false };
|
|
130
|
-
}
|
|
131
|
-
else if (typeof proxyAgentUrl === 'string') {
|
|
132
|
-
// Parse string proxy: socks5://127.0.0.1:7890
|
|
133
|
-
let protocolMatch = proxyAgentUrl.match(/^(http|https|socks5)/);
|
|
134
|
-
let protocol = protocolMatch ? protocolMatch[1] : 'http';
|
|
135
|
-
let hostMatch = proxyAgentUrl.match(/:\/\/([^:\/]+)/);
|
|
136
|
-
let host = hostMatch ? hostMatch[1] : '';
|
|
137
|
-
let portMatch = proxyAgentUrl.match(/:(\d+)/);
|
|
138
|
-
let port = portMatch ? parseInt(portMatch[1]) : 7890;
|
|
139
|
-
let proxyAgentObj = { enabled: true, protocol, host, port };
|
|
140
|
-
// Use auth from options if provided
|
|
141
|
-
if (auth) {
|
|
142
|
-
let [username, password] = auth.split("/");
|
|
143
|
-
proxyAgentObj.auth = { username, password };
|
|
144
|
-
}
|
|
145
|
-
json.proxyAgent = proxyAgentObj;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
return json;
|
|
149
|
-
}
|
|
150
|
-
const mergeProxyAgent = (argProxy, configProxy, config) => {
|
|
151
|
-
// 打印调试信息
|
|
152
|
-
(0, logger_1.debug)(config, `合并代理配置 - argProxy: ${JSON.stringify(argProxy)}, configProxy.enabled: ${configProxy?.enabled}`, 'proxy merge debug', 'details');
|
|
153
|
-
// 1. Explicit disable in Args (必须是明确设置为 false)
|
|
154
|
-
if (argProxy?.enabled === false) {
|
|
155
|
-
(0, logger_1.debug)(config, `订阅明确禁用代理`, 'proxy merge', 'details');
|
|
156
|
-
return { enabled: false };
|
|
157
|
-
}
|
|
158
|
-
// 2. Arg 有完整的 proxy 配置 (enabled=true 且有 host) -> 使用 Arg
|
|
159
|
-
if (argProxy?.enabled === true && argProxy?.host) {
|
|
160
|
-
(0, logger_1.debug)(config, `使用订阅的代理配置`, 'proxy merge', 'details');
|
|
161
|
-
return argProxy;
|
|
162
|
-
}
|
|
163
|
-
// 3. Arg 是空对象、undefined、null,或者没有 enabled 字段 -> 使用全局配置
|
|
164
|
-
// 这是关键:如果订阅没有单独配置代理,就应该使用全局配置
|
|
165
|
-
const shouldUseConfigProxy = !argProxy || Object.keys(argProxy || {}).length === 0 || argProxy?.enabled === undefined || argProxy?.enabled === null;
|
|
166
|
-
if (shouldUseConfigProxy) {
|
|
167
|
-
if (configProxy?.enabled) {
|
|
168
|
-
const result = {
|
|
169
|
-
enabled: true,
|
|
170
|
-
protocol: configProxy.protocol,
|
|
171
|
-
host: configProxy.host,
|
|
172
|
-
port: configProxy.port,
|
|
173
|
-
auth: configProxy.auth?.enabled ? configProxy.auth : undefined
|
|
174
|
-
};
|
|
175
|
-
(0, logger_1.debug)(config, `使用全局代理: ${result.protocol}://${result.host}:${result.port}`, 'proxy merge', 'info');
|
|
176
|
-
return result;
|
|
177
|
-
}
|
|
178
|
-
else {
|
|
179
|
-
(0, logger_1.debug)(config, `全局代理未启用`, 'proxy merge', 'details');
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
// 4. Arg 的 enabled=true 但没有 host -> 尝试补充全局配置
|
|
183
|
-
if (argProxy?.enabled === true && !argProxy?.host) {
|
|
184
|
-
const result = {
|
|
185
|
-
...configProxy,
|
|
186
|
-
...argProxy,
|
|
187
|
-
auth: configProxy?.auth?.enabled ? configProxy.auth : undefined
|
|
188
|
-
};
|
|
189
|
-
(0, logger_1.debug)(config, `订阅代理配置不完整,补充全局配置`, 'proxy merge', 'details');
|
|
190
|
-
return result;
|
|
191
|
-
}
|
|
192
|
-
// 5. Default disabled
|
|
193
|
-
(0, logger_1.debug)(config, `代理未配置,使用默认(禁用)`, 'proxy merge', 'details');
|
|
194
|
-
return { enabled: false };
|
|
195
|
-
};
|
|
196
|
-
const mergeProxyAgentWithLog = (argProxy, configProxy, config) => {
|
|
197
|
-
const result = mergeProxyAgent(argProxy, configProxy, config);
|
|
198
|
-
(0, logger_1.debug)(config, `[DEBUG_PROXY] mergeProxyAgent input: arg=${JSON.stringify(argProxy)} conf=${JSON.stringify(configProxy)} output=${JSON.stringify(result)}`, 'proxy merge', 'details');
|
|
199
|
-
return result;
|
|
200
|
-
};
|
|
201
|
-
function mixinArg(arg, config) {
|
|
202
|
-
const mergedProxy = mergeProxyAgentWithLog(arg?.proxyAgent, config.net?.proxyAgent, config);
|
|
203
|
-
// 打印代理配置合并结果(方便调试)
|
|
204
|
-
if (mergedProxy?.enabled) {
|
|
205
|
-
(0, logger_1.debug)(config, `使用代理: ${mergedProxy.protocol}://${mergedProxy.host}:${mergedProxy.port}`, 'proxy merge', 'details');
|
|
57
|
+
function shouldSkipByInterval(rssItem, arg, originalArg) {
|
|
58
|
+
if (!rssItem.arg.interval)
|
|
59
|
+
return false;
|
|
60
|
+
const now = Date.now();
|
|
61
|
+
const nextUpdateTime = (0, legacy_config_1.getNextUpdateTime)(arg);
|
|
62
|
+
if (nextUpdateTime && nextUpdateTime > now)
|
|
63
|
+
return true;
|
|
64
|
+
if (nextUpdateTime) {
|
|
65
|
+
const missed = Math.ceil((now - nextUpdateTime) / arg.interval);
|
|
66
|
+
(0, legacy_config_1.setNextUpdateTime)(originalArg, nextUpdateTime + (arg.interval * (missed || 1)));
|
|
206
67
|
}
|
|
207
68
|
else {
|
|
208
|
-
(0,
|
|
69
|
+
(0, legacy_config_1.setNextUpdateTime)(originalArg, now + arg.interval);
|
|
209
70
|
}
|
|
210
|
-
|
|
211
|
-
// We explicitly take known safe config sections
|
|
212
|
-
const baseConfig = {
|
|
213
|
-
...config.basic,
|
|
214
|
-
// Add other flat config sections if necessary
|
|
215
|
-
};
|
|
216
|
-
const res = {
|
|
217
|
-
...baseConfig,
|
|
218
|
-
...arg, // Args override basic config
|
|
219
|
-
filter: [...(config.msg?.keywordFilter || []), ...(arg?.filter || [])],
|
|
220
|
-
block: [...(config.msg?.keywordBlock || []), ...(arg?.block || [])],
|
|
221
|
-
template: arg.template ?? config.basic?.defaultTemplate,
|
|
222
|
-
proxyAgent: mergedProxy
|
|
223
|
-
};
|
|
224
|
-
(0, logger_1.debug)(config, `[DEBUG_PROXY] mixinArg return: ${JSON.stringify(res.proxyAgent)}`, 'mixin', 'details');
|
|
225
|
-
return res;
|
|
71
|
+
return false;
|
|
226
72
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
* 1. 抓取 RSS 数据
|
|
230
|
-
*/
|
|
231
|
-
async function fetchRssItems(ctx, config, $http, rssItem, arg, feedDebug) {
|
|
232
|
-
const rssHubUrl = config.msg?.rssHubUrl || 'https://hub.slarker.me';
|
|
233
|
-
try {
|
|
234
|
-
const urls = rssItem.url.split("|").map((u) => (0, common_1.parseQuickUrl)(u, rssHubUrl, constants_1.quickList));
|
|
235
|
-
const fetchPromises = urls.map((url) => (0, parser_1.getRssData)(ctx, config, $http, url, arg));
|
|
236
|
-
const results = await Promise.all(fetchPromises);
|
|
237
|
-
return results.flat(1);
|
|
238
|
-
}
|
|
239
|
-
catch (err) {
|
|
240
|
-
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
241
|
-
feedDebug(`Fetch failed for ${rssItem.title}: ${normalizedError.message}`, 'feeder', 'error', {
|
|
242
|
-
stage: 'fetch',
|
|
243
|
-
});
|
|
244
|
-
(0, error_tracker_1.trackError)(normalizedError, {
|
|
245
|
-
...buildFeedLogContext(rssItem),
|
|
246
|
-
stage: 'fetch',
|
|
247
|
-
});
|
|
248
|
-
return [];
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
/**
|
|
252
|
-
* 2. 过滤关键字
|
|
253
|
-
*/
|
|
254
|
-
function filterItems(config, items, arg, feedDebug) {
|
|
255
|
-
return items.filter(item => {
|
|
256
|
-
const matchKeyword = arg.filter?.find((keyword) => new RegExp(keyword, 'im').test(item.title) || new RegExp(keyword, 'im').test(item.description));
|
|
257
|
-
if (matchKeyword) {
|
|
258
|
-
feedDebug(`filter:${matchKeyword}`, 'feeder', 'info', { matchedKeyword: matchKeyword });
|
|
259
|
-
feedDebug(item, 'filter rss item', 'info', { matchedKeyword: matchKeyword });
|
|
260
|
-
}
|
|
261
|
-
return !matchKeyword;
|
|
262
|
-
});
|
|
263
|
-
}
|
|
264
|
-
/**
|
|
265
|
-
* 3. 检查更新(时间+内容)
|
|
266
|
-
*/
|
|
267
|
-
function checkForUpdates(config, rssItem, items, arg, feedDebug) {
|
|
268
|
-
// 按时间排序
|
|
269
|
-
let itemArray = items
|
|
270
|
-
.sort((a, b) => (0, common_1.parsePubDate)(config, b.pubDate).getTime() - (0, common_1.parsePubDate)(config, a.pubDate).getTime());
|
|
271
|
-
if (itemArray.length === 0) {
|
|
272
|
-
return { newItems: [], latestPubDate: new Date(), currentContent: [] };
|
|
273
|
-
}
|
|
274
|
-
const latestItem = itemArray[0];
|
|
275
|
-
const lastPubDate = (0, common_1.parsePubDate)(config, latestItem.pubDate);
|
|
276
|
-
feedDebug(`${rssItem.title}: Latest item date=${lastPubDate.toISOString()}, DB date=${rssItem.lastPubDate ? new Date(rssItem.lastPubDate).toISOString() : 'none'}`, 'feeder', 'details');
|
|
277
|
-
// 准备去重内容
|
|
278
|
-
const currentContent = config.basic?.resendUpdataContent === 'all'
|
|
279
|
-
? itemArray.map((i) => getLastContent(i, config))
|
|
280
|
-
: [getLastContent(latestItem, config)];
|
|
281
|
-
// 反转顺序(发送顺序:最早的先发)
|
|
282
|
-
if (arg.reverse) {
|
|
283
|
-
itemArray = itemArray.reverse();
|
|
284
|
-
}
|
|
285
|
-
let rssItemArray = [];
|
|
286
|
-
if (rssItem.arg.forceLength) {
|
|
287
|
-
// 强制长度模式:忽略时间,只取 N 条
|
|
288
|
-
rssItemArray = itemArray.slice(0, rssItem.arg.forceLength);
|
|
289
|
-
feedDebug(`${rssItem.title}: Force length mode, taking ${rssItemArray.length} items`, 'feeder', 'details');
|
|
290
|
-
}
|
|
291
|
-
else {
|
|
292
|
-
// 标准模式:时间 + 内容检查
|
|
293
|
-
feedDebug(`${rssItem.title}: Checking ${itemArray.length} items for updates`, 'feeder', 'details');
|
|
294
|
-
rssItemArray = itemArray.filter((v, i) => {
|
|
295
|
-
const currentItemTime = (0, common_1.parsePubDate)(config, v.pubDate).getTime();
|
|
296
|
-
const lastTime = rssItem.lastPubDate ? (0, common_1.parsePubDate)(config, rssItem.lastPubDate).getTime() : 0;
|
|
297
|
-
feedDebug(`[${i}] ${v.title?.substring(0, 30)}: time=${new Date(currentItemTime).toISOString()} > last=${new Date(lastTime).toISOString()} ? ${currentItemTime > lastTime}`, 'feeder', 'details');
|
|
298
|
-
// 严格时间检查
|
|
299
|
-
if (currentItemTime > lastTime) {
|
|
300
|
-
feedDebug(`[${i}] ✓ Item is new (time check)`, 'feeder', 'details');
|
|
301
|
-
return true;
|
|
302
|
-
}
|
|
303
|
-
// 内容哈希检查(时间相同但内容变化)
|
|
304
|
-
if (config.basic?.resendUpdataContent !== 'disable') {
|
|
305
|
-
const newItemContent = getLastContent(v, config);
|
|
306
|
-
const oldItemMatch = rssItem.lastContent?.itemArray?.find((old) => (newItemContent.guid && old.guid === newItemContent.guid) ||
|
|
307
|
-
(old.link === newItemContent.link && old.title === newItemContent.title));
|
|
308
|
-
if (oldItemMatch) {
|
|
309
|
-
const descriptionChanged = JSON.stringify(oldItemMatch.description) !== JSON.stringify(newItemContent.description);
|
|
310
|
-
if (descriptionChanged) {
|
|
311
|
-
feedDebug(`[${i}] ✓ Item is updated (content changed)`, 'feeder', 'details');
|
|
312
|
-
}
|
|
313
|
-
else {
|
|
314
|
-
feedDebug(`[${i}] ✗ Item filtered (already sent)`, 'feeder', 'details');
|
|
315
|
-
}
|
|
316
|
-
return descriptionChanged;
|
|
317
|
-
}
|
|
318
|
-
else {
|
|
319
|
-
feedDebug(`[${i}] ✗ Item filtered (no match in lastContent)`, 'feeder', 'details');
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
feedDebug(`[${i}] ✗ Item filtered (failed all checks)`, 'feeder', 'details');
|
|
323
|
-
return false;
|
|
324
|
-
});
|
|
325
|
-
// 应用最大条目限制
|
|
326
|
-
if (arg.maxRssItem) {
|
|
327
|
-
rssItemArray = rssItemArray.slice(0, arg.maxRssItem);
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
return { newItems: rssItemArray, latestPubDate: lastPubDate, currentContent };
|
|
73
|
+
async function persistSubscriptionState(ctx, rssItemId, state) {
|
|
74
|
+
await ctx.database.set('rssOwl', { id: rssItemId }, state);
|
|
331
75
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
const itemsToSend = [...items].reverse();
|
|
337
|
-
// 生成所有消息
|
|
338
|
-
const messageList = (await Promise.all(itemsToSend.map(async (i) => await processor.parseRssItem(i, { ...rssItem, ...arg }, rssItem.author)))).filter(m => m);
|
|
339
|
-
return { messageList, itemsToSend };
|
|
340
|
-
}
|
|
341
|
-
/**
|
|
342
|
-
* 5. 构建最终消息
|
|
343
|
-
*/
|
|
344
|
-
function buildFinalMessage(config, messageList, rssItem, arg) {
|
|
345
|
-
let message = "";
|
|
346
|
-
const shouldMerge = arg.merge === true || config.basic?.merge === '一直合并' || (config.basic?.merge === '有多条更新时合并' && messageList.length > 1);
|
|
347
|
-
// 检查是否需要合并视频
|
|
348
|
-
const hasVideo = config.basic?.margeVideo && messageList.some(msg => /<video/.test(msg));
|
|
349
|
-
if (shouldMerge || hasVideo) {
|
|
350
|
-
message = `<message forward><author id="${rssItem.author}"/>${messageList.map(m => `<message>${m}</message>`).join("")}</message>`;
|
|
351
|
-
}
|
|
352
|
-
else {
|
|
353
|
-
message = messageList.join("");
|
|
354
|
-
}
|
|
355
|
-
// 添加提及
|
|
356
|
-
if (rssItem.followers && rssItem.followers.length > 0) {
|
|
357
|
-
const mentions = rssItem.followers.map((id) => `<at ${id === 'all' ? 'type="all"' : `id="${id}"`}/>`).join(" ");
|
|
358
|
-
message += `<message>${mentions}</message>`;
|
|
359
|
-
}
|
|
360
|
-
return message;
|
|
76
|
+
function buildQueueUid(item, config) {
|
|
77
|
+
return String(item?.link
|
|
78
|
+
|| item?.guid
|
|
79
|
+
|| JSON.stringify((0, feeder_runtime_1.getLastContent)(item, config)));
|
|
361
80
|
}
|
|
362
81
|
// ============ 主函数 ============
|
|
363
82
|
/**
|
|
@@ -365,78 +84,62 @@ function buildFinalMessage(config, messageList, rssItem, arg) {
|
|
|
365
84
|
*/
|
|
366
85
|
async function feeder(deps, processor) {
|
|
367
86
|
const { ctx, config, $http, queueManager } = deps;
|
|
368
|
-
// Use type assertion for custom table
|
|
369
87
|
const rssList = await ctx.database.get('rssOwl', {});
|
|
370
88
|
if (!rssList || rssList.length === 0)
|
|
371
89
|
return;
|
|
372
90
|
for (const rssItem of rssList) {
|
|
373
91
|
try {
|
|
374
|
-
const feedDebug = createFeedDebug(config, rssItem);
|
|
375
|
-
|
|
376
|
-
let arg = mixinArg(rssItem.arg || {}, config);
|
|
92
|
+
const feedDebug = (0, feeder_runtime_1.createFeedDebug)(config, rssItem);
|
|
93
|
+
const arg = (0, feeder_arg_1.mixinArg)(rssItem.arg || {}, config);
|
|
377
94
|
feedDebug(`[DEBUG_PROXY] feeder mixinArg result proxyAgent: ${JSON.stringify(arg.proxyAgent)}`, 'feeder', 'details');
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
if (arg.nextUpdataTime && arg.nextUpdataTime > now)
|
|
383
|
-
continue;
|
|
384
|
-
// Calculate next update time
|
|
385
|
-
if (arg.nextUpdataTime) {
|
|
386
|
-
const missed = Math.ceil((now - arg.nextUpdataTime) / arg.interval);
|
|
387
|
-
originalArg.nextUpdataTime = arg.nextUpdataTime + (arg.interval * (missed || 1));
|
|
388
|
-
}
|
|
389
|
-
else {
|
|
390
|
-
originalArg.nextUpdataTime = now + arg.interval;
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
// 3. Fetch RSS Data
|
|
394
|
-
const rssItemList = await fetchRssItems(ctx, config, $http, rssItem, arg, feedDebug);
|
|
95
|
+
const originalArg = (0, legacy_config_1.normalizeSubscriptionArg)((0, koishi_1.clone)(rssItem.arg || {}));
|
|
96
|
+
if (shouldSkipByInterval(rssItem, arg, originalArg))
|
|
97
|
+
continue;
|
|
98
|
+
const rssItemList = await (0, feeder_runtime_1.fetchRssItems)(ctx, config, $http, rssItem, arg, feedDebug);
|
|
395
99
|
if (rssItemList.length === 0) {
|
|
396
|
-
await ctx
|
|
100
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
397
101
|
lastPubDate: rssItem.lastPubDate,
|
|
398
102
|
arg: originalArg,
|
|
399
|
-
lastContent: rssItem.lastContent || { itemArray: [] }
|
|
103
|
+
lastContent: rssItem.lastContent || { itemArray: [] },
|
|
400
104
|
});
|
|
401
105
|
continue;
|
|
402
106
|
}
|
|
403
|
-
|
|
404
|
-
const filteredItems = filterItems(config, rssItemList, arg, feedDebug);
|
|
107
|
+
const filteredItems = (0, feeder_runtime_1.filterItems)(rssItemList, arg, feedDebug);
|
|
405
108
|
if (filteredItems.length === 0) {
|
|
406
109
|
const latestItem = [...rssItemList]
|
|
407
110
|
.sort((a, b) => (0, common_1.parsePubDate)(config, b.pubDate).getTime() - (0, common_1.parsePubDate)(config, a.pubDate).getTime())[0];
|
|
408
|
-
await ctx
|
|
111
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
409
112
|
lastPubDate: latestItem ? (0, common_1.parsePubDate)(config, latestItem.pubDate) : rssItem.lastPubDate,
|
|
410
113
|
arg: originalArg,
|
|
411
114
|
lastContent: latestItem
|
|
412
|
-
? { itemArray: [getLastContent(latestItem, config)] }
|
|
413
|
-
: (rssItem.lastContent || { itemArray: [] })
|
|
115
|
+
? { itemArray: [(0, feeder_runtime_1.getLastContent)(latestItem, config)] }
|
|
116
|
+
: (rssItem.lastContent || { itemArray: [] }),
|
|
414
117
|
});
|
|
415
118
|
continue;
|
|
416
119
|
}
|
|
417
|
-
|
|
418
|
-
const { newItems, latestPubDate, currentContent } = checkForUpdates(config, rssItem, filteredItems, arg, feedDebug);
|
|
120
|
+
const { newItems, latestPubDate, currentContent } = (0, feeder_runtime_1.checkForUpdates)(config, rssItem, filteredItems, arg, feedDebug);
|
|
419
121
|
if (newItems.length === 0) {
|
|
420
122
|
feedDebug(`${rssItem.title}: No new items found after filtering`, 'feeder', 'info', { newItemCount: 0 });
|
|
421
|
-
await ctx
|
|
123
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
422
124
|
lastPubDate: latestPubDate,
|
|
423
125
|
arg: originalArg,
|
|
424
|
-
lastContent: { itemArray: currentContent }
|
|
126
|
+
lastContent: { itemArray: currentContent },
|
|
425
127
|
});
|
|
426
128
|
continue;
|
|
427
129
|
}
|
|
428
130
|
feedDebug(`${rssItem.title}: Found ${newItems.length} new items`, 'feeder', 'info', { newItemCount: newItems.length });
|
|
429
131
|
feedDebug(newItems.map(i => i.title), 'feeder', 'info', { newItemCount: newItems.length });
|
|
430
|
-
|
|
431
|
-
const { messageList, itemsToSend } = await generateMessages(processor, newItems, rssItem, arg);
|
|
132
|
+
const { messageList, itemsToSend } = await (0, feeder_runtime_1.generateMessages)(processor, newItems, rssItem, arg);
|
|
432
133
|
if (messageList.length === 0) {
|
|
433
134
|
feedDebug(`${rssItem.title}: Items found but parsed to empty messages`, 'feeder', 'info', { newItemCount: newItems.length });
|
|
434
|
-
await ctx
|
|
135
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
136
|
+
lastPubDate: latestPubDate,
|
|
137
|
+
arg: originalArg,
|
|
138
|
+
lastContent: { itemArray: currentContent },
|
|
139
|
+
});
|
|
435
140
|
continue;
|
|
436
141
|
}
|
|
437
|
-
|
|
438
|
-
const message = buildFinalMessage(config, messageList, rssItem, arg);
|
|
439
|
-
// 8. Add to Queue
|
|
142
|
+
const message = (0, feeder_runtime_1.buildFinalMessage)(config, messageList, rssItem, arg);
|
|
440
143
|
const taskContent = {
|
|
441
144
|
message,
|
|
442
145
|
originalItem: itemsToSend[0],
|
|
@@ -445,29 +148,28 @@ async function feeder(deps, processor) {
|
|
|
445
148
|
description: itemsToSend[0]?.description,
|
|
446
149
|
link: itemsToSend[0]?.link,
|
|
447
150
|
pubDate: (0, common_1.parsePubDate)(config, itemsToSend[0]?.pubDate),
|
|
448
|
-
imageUrl: itemsToSend[0]?.enclosure?.url
|
|
151
|
+
imageUrl: itemsToSend[0]?.enclosure?.url,
|
|
449
152
|
};
|
|
450
153
|
await queueManager.addTask({
|
|
451
154
|
subscribeId: String(rssItem.id),
|
|
452
|
-
rssId: rssItem.rssId || rssItem.title,
|
|
453
|
-
uid: itemsToSend[0]
|
|
155
|
+
rssId: String(rssItem.rssId || rssItem.title),
|
|
156
|
+
uid: buildQueueUid(itemsToSend[0], config),
|
|
454
157
|
guildId: rssItem.guildId,
|
|
455
158
|
platform: rssItem.platform,
|
|
456
|
-
content: taskContent
|
|
159
|
+
content: taskContent,
|
|
457
160
|
});
|
|
458
161
|
feedDebug(`✓ 已添加到发送队列: ${rssItem.title}`, 'feeder', 'info', {
|
|
459
162
|
queuedItemTitle: itemsToSend[0]?.title,
|
|
460
163
|
});
|
|
461
|
-
|
|
462
|
-
await ctx.database.set('rssOwl', { id: rssItem.id }, {
|
|
164
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
463
165
|
lastPubDate: latestPubDate,
|
|
464
166
|
arg: originalArg,
|
|
465
|
-
lastContent: { itemArray: currentContent }
|
|
167
|
+
lastContent: { itemArray: currentContent },
|
|
466
168
|
});
|
|
467
169
|
}
|
|
468
170
|
catch (err) {
|
|
469
171
|
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
470
|
-
const feedContext = buildFeedLogContext(rssItem);
|
|
172
|
+
const feedContext = (0, feeder_runtime_1.buildFeedLogContext)(rssItem);
|
|
471
173
|
(0, logger_1.debug)(config, `Feeder error for ${rssItem.url}: ${normalizedError.message}`, 'feeder', 'error', feedContext);
|
|
472
174
|
(0, error_tracker_1.trackError)(normalizedError, feedContext);
|
|
473
175
|
}
|
|
@@ -476,6 +178,7 @@ async function feeder(deps, processor) {
|
|
|
476
178
|
function startFeeder(ctx, config, $http, processor, queueManager) {
|
|
477
179
|
const deps = { ctx, config, $http, queueManager };
|
|
478
180
|
const lifecycleDebug = (0, logger_1.createDebugWithContext)(config, { lifecycle: 'feeder' });
|
|
181
|
+
const queueRuntimeConfig = (0, notification_queue_retry_1.getQueueRuntimeConfig)(config);
|
|
479
182
|
// Initial run
|
|
480
183
|
feeder(deps, processor).catch(err => {
|
|
481
184
|
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
@@ -498,7 +201,7 @@ function startFeeder(ctx, config, $http, processor, queueManager) {
|
|
|
498
201
|
}, refreshInterval);
|
|
499
202
|
// 启动消费者定时器(处理发送队列)
|
|
500
203
|
// 频率更高,确保消息快速发送
|
|
501
|
-
const queueProcessInterval =
|
|
204
|
+
const queueProcessInterval = queueRuntimeConfig.processIntervalSeconds * 1000;
|
|
502
205
|
queueInterval = setInterval(async () => {
|
|
503
206
|
await queueManager.processQueue();
|
|
504
207
|
}, queueProcessInterval);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import * as cheerio from 'cheerio';
|
|
2
|
+
import { Context } from 'koishi';
|
|
3
|
+
import { Config, rssArg } from '../types';
|
|
4
|
+
export interface ItemProcessorRuntimeDeps {
|
|
5
|
+
ctx: Context;
|
|
6
|
+
config: Config;
|
|
7
|
+
$http: any;
|
|
8
|
+
}
|
|
9
|
+
interface RenderDescriptionOptions {
|
|
10
|
+
contentStyle?: string;
|
|
11
|
+
dividerStyle?: string;
|
|
12
|
+
logImageMode?: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* 标准化 RSS 字段内容,统一数组 / 空值 / 非字符串输入。
|
|
16
|
+
*/
|
|
17
|
+
export declare function normalizeText(value: unknown): string;
|
|
18
|
+
/**
|
|
19
|
+
* 解析并去重 HTML 中的图片资源,返回原图到最终地址的映射表。
|
|
20
|
+
*/
|
|
21
|
+
export declare function buildResolvedImageMap(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg): Promise<Record<string, string>>;
|
|
22
|
+
/**
|
|
23
|
+
* 将 HTML 中的唯一图片列表解析为消息图片串。
|
|
24
|
+
*/
|
|
25
|
+
export declare function renderImageListFromHtml(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg): Promise<string>;
|
|
26
|
+
/**
|
|
27
|
+
* 将已加载的 HTML 内容渲染为最终消息。
|
|
28
|
+
*/
|
|
29
|
+
export declare function renderLoadedHtml(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg, useXml?: boolean): Promise<string>;
|
|
30
|
+
/**
|
|
31
|
+
* 渲染模板描述正文,并在图片模式下按需注入 AI 摘要。
|
|
32
|
+
*/
|
|
33
|
+
export declare function renderTemplatedDescription(deps: ItemProcessorRuntimeDeps, item: any, arg: rssArg, description: string, options?: RenderDescriptionOptions): Promise<string>;
|
|
34
|
+
/**
|
|
35
|
+
* 提取 HTML 中的视频资源,并按配置补齐 poster 图。
|
|
36
|
+
*/
|
|
37
|
+
export declare function processVideos(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg, videoList: Array<[string, string]>): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* 将视频列表格式化为最终消息片段。
|
|
40
|
+
*/
|
|
41
|
+
export declare function formatVideoList(videoList: Array<[string, string]>): string;
|
|
42
|
+
/**
|
|
43
|
+
* 统一执行图片渲染,兼容 base64 / File / assets / HTML 回退。
|
|
44
|
+
*/
|
|
45
|
+
export declare function renderImage(htmlContent: string, deps: ItemProcessorRuntimeDeps, arg: rssArg): Promise<string>;
|
|
46
|
+
export {};
|