@anyul/koishi-plugin-rss 5.2.2 → 5.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -37
- package/lib/commands/error-handler.js +13 -4
- package/lib/commands/index.d.ts +20 -1
- package/lib/commands/index.js +394 -2
- package/lib/commands/runtime.d.ts +17 -0
- package/lib/commands/runtime.js +27 -0
- package/lib/commands/subscription-create.d.ts +23 -0
- package/lib/commands/subscription-create.js +145 -0
- package/lib/commands/subscription-edit.d.ts +7 -0
- package/lib/commands/subscription-edit.js +177 -0
- package/lib/commands/subscription-management.d.ts +12 -0
- package/lib/commands/subscription-management.js +176 -0
- package/lib/commands/utils.d.ts +13 -1
- package/lib/commands/utils.js +43 -2
- package/lib/commands/web-monitor.d.ts +15 -0
- package/lib/commands/web-monitor.js +222 -0
- package/lib/config.js +25 -0
- package/lib/constants.d.ts +1 -1
- package/lib/constants.js +46 -83
- package/lib/core/ai-cache.d.ts +27 -0
- package/lib/core/ai-cache.js +169 -0
- package/lib/core/ai-client.d.ts +12 -0
- package/lib/core/ai-client.js +65 -0
- package/lib/core/ai-selector.d.ts +2 -0
- package/lib/core/ai-selector.js +80 -0
- package/lib/core/ai-summary.d.ts +10 -0
- package/lib/core/ai-summary.js +73 -0
- package/lib/core/ai-utils.d.ts +10 -0
- package/lib/core/ai-utils.js +104 -0
- package/lib/core/ai.d.ts +3 -77
- package/lib/core/ai.js +13 -455
- package/lib/core/feeder-arg.d.ts +17 -0
- package/lib/core/feeder-arg.js +234 -0
- package/lib/core/feeder-runtime.d.ts +96 -0
- package/lib/core/feeder-runtime.js +233 -0
- package/lib/core/feeder.d.ts +4 -6
- package/lib/core/feeder.js +120 -304
- package/lib/core/item-processor-runtime.d.ts +46 -0
- package/lib/core/item-processor-runtime.js +215 -0
- package/lib/core/item-processor-template.d.ts +16 -0
- package/lib/core/item-processor-template.js +158 -0
- package/lib/core/item-processor.d.ts +1 -10
- package/lib/core/item-processor.js +48 -393
- package/lib/core/notification-queue-retry.d.ts +25 -0
- package/lib/core/notification-queue-retry.js +78 -0
- package/lib/core/notification-queue-sender.d.ts +20 -0
- package/lib/core/notification-queue-sender.js +118 -0
- package/lib/core/notification-queue-store.d.ts +19 -0
- package/lib/core/notification-queue-store.js +137 -0
- package/lib/core/notification-queue-types.d.ts +49 -0
- package/lib/core/notification-queue-types.js +2 -0
- package/lib/core/notification-queue.d.ts +13 -72
- package/lib/core/notification-queue.js +132 -262
- package/lib/core/parser.js +12 -0
- package/lib/core/renderer.d.ts +15 -0
- package/lib/core/renderer.js +91 -23
- package/lib/core/search-format.d.ts +3 -0
- package/lib/core/search-format.js +36 -0
- package/lib/core/search-providers.d.ts +13 -0
- package/lib/core/search-providers.js +175 -0
- package/lib/core/search-rotation.d.ts +4 -0
- package/lib/core/search-rotation.js +55 -0
- package/lib/core/search-service.d.ts +3 -0
- package/lib/core/search-service.js +100 -0
- package/lib/core/search-types.d.ts +39 -0
- package/lib/core/search-types.js +2 -0
- package/lib/core/search.d.ts +4 -101
- package/lib/core/search.js +10 -508
- package/lib/index.js +50 -1160
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/lib/types.d.ts +51 -6
- package/lib/utils/common.js +52 -3
- package/lib/utils/error-handler.d.ts +8 -0
- package/lib/utils/error-handler.js +27 -0
- package/lib/utils/error-tracker.js +24 -8
- package/lib/utils/fetcher.js +68 -9
- package/lib/utils/legacy-config.d.ts +12 -0
- package/lib/utils/legacy-config.js +56 -0
- package/lib/utils/logger.d.ts +4 -2
- package/lib/utils/logger.js +193 -34
- package/lib/utils/media.js +3 -6
- package/lib/utils/proxy.d.ts +3 -0
- package/lib/utils/proxy.js +14 -0
- package/lib/utils/sanitizer.d.ts +58 -0
- package/lib/utils/sanitizer.js +227 -0
- package/lib/utils/security.d.ts +75 -0
- package/lib/utils/security.js +312 -0
- package/lib/utils/structured-logger.d.ts +7 -3
- package/lib/utils/structured-logger.js +29 -39
- package/package.json +2 -1
package/lib/core/feeder.js
CHANGED
|
@@ -33,328 +33,113 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.findRssItem =
|
|
37
|
-
exports.getLastContent = getLastContent;
|
|
38
|
-
exports.formatArg = formatArg;
|
|
39
|
-
exports.mixinArg = mixinArg;
|
|
36
|
+
exports.getLastContent = exports.findRssItem = exports.mixinArg = exports.formatArg = void 0;
|
|
40
37
|
exports.feeder = feeder;
|
|
41
38
|
exports.startFeeder = startFeeder;
|
|
42
39
|
exports.stopFeeder = stopFeeder;
|
|
43
40
|
const koishi_1 = require("koishi");
|
|
44
|
-
const logger_1 = require("../utils/logger");
|
|
45
41
|
const common_1 = require("../utils/common");
|
|
46
|
-
const
|
|
47
|
-
const
|
|
42
|
+
const error_handler_1 = require("../utils/error-handler");
|
|
43
|
+
const error_tracker_1 = require("../utils/error-tracker");
|
|
44
|
+
const legacy_config_1 = require("../utils/legacy-config");
|
|
45
|
+
const logger_1 = require("../utils/logger");
|
|
46
|
+
const feeder_arg_1 = require("./feeder-arg");
|
|
47
|
+
const feeder_runtime_1 = require("./feeder-runtime");
|
|
48
|
+
const notification_queue_retry_1 = require("./notification-queue-retry");
|
|
49
|
+
var feeder_arg_2 = require("./feeder-arg");
|
|
50
|
+
Object.defineProperty(exports, "formatArg", { enumerable: true, get: function () { return feeder_arg_2.formatArg; } });
|
|
51
|
+
Object.defineProperty(exports, "mixinArg", { enumerable: true, get: function () { return feeder_arg_2.mixinArg; } });
|
|
52
|
+
var feeder_runtime_2 = require("./feeder-runtime");
|
|
53
|
+
Object.defineProperty(exports, "findRssItem", { enumerable: true, get: function () { return feeder_runtime_2.findRssItem; } });
|
|
54
|
+
Object.defineProperty(exports, "getLastContent", { enumerable: true, get: function () { return feeder_runtime_2.getLastContent; } });
|
|
48
55
|
let interval = null;
|
|
49
56
|
let queueInterval = null;
|
|
50
|
-
function
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
(rssList.findIndex(i => i.url == keyword) + 1) ||
|
|
61
|
-
(rssList.findIndex(i => i.url.indexOf(keyword) + 1) + 1) ||
|
|
62
|
-
(rssList.findIndex(i => i.title.indexOf(keyword) + 1) + 1)) - 1;
|
|
63
|
-
return rssList[index];
|
|
64
|
-
}
|
|
65
|
-
function getLastContent(item, config) {
|
|
66
|
-
let arr = ['title', 'description', 'link', 'guid'];
|
|
67
|
-
let obj = Object.assign({}, ...arr.map(i => (0, koishi_1.clone)(item?.[i]) ? { [i]: item[i] } : {}));
|
|
68
|
-
return { ...obj, description: String(obj?.description).replaceAll(/\s/g, '') };
|
|
69
|
-
}
|
|
70
|
-
function formatArg(options, config) {
|
|
71
|
-
let { arg, template, auth } = options;
|
|
72
|
-
// 特殊处理:提取完整的 proxyAgent URL
|
|
73
|
-
let proxyAgentUrl;
|
|
74
|
-
if (arg && arg.includes('proxyAgent:')) {
|
|
75
|
-
const match = arg.match(/proxyAgent:([^,]+)/);
|
|
76
|
-
if (match) {
|
|
77
|
-
proxyAgentUrl = match[1];
|
|
78
|
-
// 从 arg 中移除 proxyAgent,避免被 split(":") 破坏
|
|
79
|
-
arg = arg.replace(/proxyAgent:[^,]+/, '').replace(/^,|,$/g, '').replace(/,,/g, ',');
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
let json = Object.assign({}, ...(arg?.split(',')?.map((i) => ({ [i.split(":")[0]]: i.split(":")[1] })) || []));
|
|
83
|
-
let key = ["forceLength", "reverse", "timeout", "interval", "merge", "maxRssItem", "firstLoad", "bodyWidth", "bodyPadding", "filter", "block"];
|
|
84
|
-
let booleanKey = ['firstLoad', "reverse", 'merge'];
|
|
85
|
-
let numberKey = ['forceLength', "timeout", 'interval', 'maxRssItem', 'bodyWidth', 'bodyPadding'];
|
|
86
|
-
let falseContent = ['false', 'null', ''];
|
|
87
|
-
json = Object.assign({}, ...Object.keys(json).filter((i) => key.some((key) => key == i)).map((key) => ({ [key]: booleanKey.some((bkey) => bkey == key) ? !falseContent.some((c) => c == json[key]) : numberKey.some((nkey) => nkey == key) ? (+json[key]) : json[key] })));
|
|
88
|
-
if (template && config.template) {
|
|
89
|
-
json['template'] = template;
|
|
90
|
-
}
|
|
91
|
-
// Date/Number conversions
|
|
92
|
-
if (json.interval)
|
|
93
|
-
json.interval = parseInt(json.interval) * 1000;
|
|
94
|
-
if (json.forceLength)
|
|
95
|
-
json.forceLength = parseInt(json.forceLength);
|
|
96
|
-
// Array conversions
|
|
97
|
-
if (json.filter && typeof json.filter === 'string')
|
|
98
|
-
json.filter = json.filter.split("/");
|
|
99
|
-
if (json.block && typeof json.block === 'string')
|
|
100
|
-
json.block = json.block.split("/");
|
|
101
|
-
// Proxy Argument Parsing (使用提取的完整 URL)
|
|
102
|
-
if (proxyAgentUrl) {
|
|
103
|
-
if (['false', 'none', ''].includes(String(proxyAgentUrl))) {
|
|
104
|
-
json.proxyAgent = { enabled: false };
|
|
105
|
-
}
|
|
106
|
-
else if (typeof proxyAgentUrl === 'string') {
|
|
107
|
-
// Parse string proxy: socks5://127.0.0.1:7890
|
|
108
|
-
let protocolMatch = proxyAgentUrl.match(/^(http|https|socks5)/);
|
|
109
|
-
let protocol = protocolMatch ? protocolMatch[1] : 'http';
|
|
110
|
-
let hostMatch = proxyAgentUrl.match(/:\/\/([^:\/]+)/);
|
|
111
|
-
let host = hostMatch ? hostMatch[1] : '';
|
|
112
|
-
let portMatch = proxyAgentUrl.match(/:(\d+)/);
|
|
113
|
-
let port = portMatch ? parseInt(portMatch[1]) : 7890;
|
|
114
|
-
let proxyAgentObj = { enabled: true, protocol, host, port };
|
|
115
|
-
// Use auth from options if provided
|
|
116
|
-
if (auth) {
|
|
117
|
-
let [username, password] = auth.split("/");
|
|
118
|
-
proxyAgentObj.auth = { username, password };
|
|
119
|
-
}
|
|
120
|
-
json.proxyAgent = proxyAgentObj;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
return json;
|
|
124
|
-
}
|
|
125
|
-
const mergeProxyAgent = (argProxy, configProxy, config) => {
|
|
126
|
-
// 打印调试信息
|
|
127
|
-
(0, logger_1.debug)(config, `合并代理配置 - argProxy: ${JSON.stringify(argProxy)}, configProxy.enabled: ${configProxy?.enabled}`, 'proxy merge debug', 'details');
|
|
128
|
-
// 1. Explicit disable in Args (必须是明确设置为 false)
|
|
129
|
-
if (argProxy?.enabled === false) {
|
|
130
|
-
(0, logger_1.debug)(config, `订阅明确禁用代理`, 'proxy merge', 'details');
|
|
131
|
-
return { enabled: false };
|
|
132
|
-
}
|
|
133
|
-
// 2. Arg 有完整的 proxy 配置 (enabled=true 且有 host) -> 使用 Arg
|
|
134
|
-
if (argProxy?.enabled === true && argProxy?.host) {
|
|
135
|
-
(0, logger_1.debug)(config, `使用订阅的代理配置`, 'proxy merge', 'details');
|
|
136
|
-
return argProxy;
|
|
137
|
-
}
|
|
138
|
-
// 3. Arg 是空对象、undefined、null,或者没有 enabled 字段 -> 使用全局配置
|
|
139
|
-
// 这是关键:如果订阅没有单独配置代理,就应该使用全局配置
|
|
140
|
-
const shouldUseConfigProxy = !argProxy || Object.keys(argProxy || {}).length === 0 || argProxy?.enabled === undefined || argProxy?.enabled === null;
|
|
141
|
-
if (shouldUseConfigProxy) {
|
|
142
|
-
if (configProxy?.enabled) {
|
|
143
|
-
const result = {
|
|
144
|
-
enabled: true,
|
|
145
|
-
protocol: configProxy.protocol,
|
|
146
|
-
host: configProxy.host,
|
|
147
|
-
port: configProxy.port,
|
|
148
|
-
auth: configProxy.auth?.enabled ? configProxy.auth : undefined
|
|
149
|
-
};
|
|
150
|
-
(0, logger_1.debug)(config, `使用全局代理: ${result.protocol}://${result.host}:${result.port}`, 'proxy merge', 'info');
|
|
151
|
-
return result;
|
|
152
|
-
}
|
|
153
|
-
else {
|
|
154
|
-
(0, logger_1.debug)(config, `全局代理未启用`, 'proxy merge', 'details');
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
// 4. Arg 的 enabled=true 但没有 host -> 尝试补充全局配置
|
|
158
|
-
if (argProxy?.enabled === true && !argProxy?.host) {
|
|
159
|
-
const result = {
|
|
160
|
-
...configProxy,
|
|
161
|
-
...argProxy,
|
|
162
|
-
auth: configProxy?.auth?.enabled ? configProxy.auth : undefined
|
|
163
|
-
};
|
|
164
|
-
(0, logger_1.debug)(config, `订阅代理配置不完整,补充全局配置`, 'proxy merge', 'details');
|
|
165
|
-
return result;
|
|
166
|
-
}
|
|
167
|
-
// 5. Default disabled
|
|
168
|
-
(0, logger_1.debug)(config, `代理未配置,使用默认(禁用)`, 'proxy merge', 'details');
|
|
169
|
-
return { enabled: false };
|
|
170
|
-
};
|
|
171
|
-
const mergeProxyAgentWithLog = (argProxy, configProxy, config) => {
|
|
172
|
-
const result = mergeProxyAgent(argProxy, configProxy, config);
|
|
173
|
-
(0, logger_1.debug)(config, `[DEBUG_PROXY] mergeProxyAgent input: arg=${JSON.stringify(argProxy)} conf=${JSON.stringify(configProxy)} output=${JSON.stringify(result)}`, 'proxy merge', 'details');
|
|
174
|
-
return result;
|
|
175
|
-
};
|
|
176
|
-
function mixinArg(arg, config) {
|
|
177
|
-
const mergedProxy = mergeProxyAgentWithLog(arg?.proxyAgent, config.net?.proxyAgent, config);
|
|
178
|
-
// 打印代理配置合并结果(方便调试)
|
|
179
|
-
if (mergedProxy?.enabled) {
|
|
180
|
-
(0, logger_1.debug)(config, `使用代理: ${mergedProxy.protocol}://${mergedProxy.host}:${mergedProxy.port}`, 'proxy merge', 'details');
|
|
57
|
+
function shouldSkipByInterval(rssItem, arg, originalArg) {
|
|
58
|
+
if (!rssItem.arg.interval)
|
|
59
|
+
return false;
|
|
60
|
+
const now = Date.now();
|
|
61
|
+
const nextUpdateTime = (0, legacy_config_1.getNextUpdateTime)(arg);
|
|
62
|
+
if (nextUpdateTime && nextUpdateTime > now)
|
|
63
|
+
return true;
|
|
64
|
+
if (nextUpdateTime) {
|
|
65
|
+
const missed = Math.ceil((now - nextUpdateTime) / arg.interval);
|
|
66
|
+
(0, legacy_config_1.setNextUpdateTime)(originalArg, nextUpdateTime + (arg.interval * (missed || 1)));
|
|
181
67
|
}
|
|
182
68
|
else {
|
|
183
|
-
(0,
|
|
69
|
+
(0, legacy_config_1.setNextUpdateTime)(originalArg, now + arg.interval);
|
|
184
70
|
}
|
|
185
|
-
|
|
186
|
-
// We explicitly take known safe config sections
|
|
187
|
-
const baseConfig = {
|
|
188
|
-
...config.basic,
|
|
189
|
-
// Add other flat config sections if necessary
|
|
190
|
-
};
|
|
191
|
-
const res = {
|
|
192
|
-
...baseConfig,
|
|
193
|
-
...arg, // Args override basic config
|
|
194
|
-
filter: [...(config.msg?.keywordFilter || []), ...(arg?.filter || [])],
|
|
195
|
-
block: [...(config.msg?.keywordBlock || []), ...(arg?.block || [])],
|
|
196
|
-
template: arg.template ?? config.basic?.defaultTemplate,
|
|
197
|
-
proxyAgent: mergedProxy
|
|
198
|
-
};
|
|
199
|
-
(0, logger_1.debug)(config, `[DEBUG_PROXY] mixinArg return: ${JSON.stringify(res.proxyAgent)}`, 'mixin', 'details');
|
|
200
|
-
return res;
|
|
71
|
+
return false;
|
|
201
72
|
}
|
|
73
|
+
async function persistSubscriptionState(ctx, rssItemId, state) {
|
|
74
|
+
await ctx.database.set('rssOwl', { id: rssItemId }, state);
|
|
75
|
+
}
|
|
76
|
+
function buildQueueUid(item, config) {
|
|
77
|
+
return String(item?.link
|
|
78
|
+
|| item?.guid
|
|
79
|
+
|| JSON.stringify((0, feeder_runtime_1.getLastContent)(item, config)));
|
|
80
|
+
}
|
|
81
|
+
// ============ 主函数 ============
|
|
202
82
|
/**
|
|
203
83
|
* 生产者:抓取 RSS,发现新消息,存入队列
|
|
204
84
|
*/
|
|
205
85
|
async function feeder(deps, processor) {
|
|
206
86
|
const { ctx, config, $http, queueManager } = deps;
|
|
207
|
-
// Use type assertion for custom table
|
|
208
87
|
const rssList = await ctx.database.get('rssOwl', {});
|
|
209
88
|
if (!rssList || rssList.length === 0)
|
|
210
89
|
return;
|
|
211
90
|
for (const rssItem of rssList) {
|
|
212
91
|
try {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
if (rssItem.arg.interval) {
|
|
219
|
-
const now = Date.now();
|
|
220
|
-
if (arg.nextUpdataTime && arg.nextUpdataTime > now)
|
|
221
|
-
continue;
|
|
222
|
-
// Calculate next update time
|
|
223
|
-
if (arg.nextUpdataTime) {
|
|
224
|
-
const missed = Math.ceil((now - arg.nextUpdataTime) / arg.interval);
|
|
225
|
-
originalArg.nextUpdataTime = arg.nextUpdataTime + (arg.interval * (missed || 1));
|
|
226
|
-
}
|
|
227
|
-
else {
|
|
228
|
-
originalArg.nextUpdataTime = now + arg.interval;
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
// 3. Fetch RSS Data
|
|
232
|
-
// Use config.msg.rssHubUrl for quick url parsing
|
|
233
|
-
const rssHubUrl = config.msg?.rssHubUrl || 'https://hub.slarker.me';
|
|
234
|
-
let rssItemList = [];
|
|
235
|
-
try {
|
|
236
|
-
const urls = rssItem.url.split("|").map((u) => (0, common_1.parseQuickUrl)(u, rssHubUrl, constants_1.quickList));
|
|
237
|
-
const fetchPromises = urls.map((url) => (0, parser_1.getRssData)(ctx, config, $http, url, arg));
|
|
238
|
-
const results = await Promise.all(fetchPromises);
|
|
239
|
-
rssItemList = results.flat(1);
|
|
240
|
-
}
|
|
241
|
-
catch (err) {
|
|
242
|
-
(0, logger_1.debug)(config, `Fetch failed for ${rssItem.title}: ${err.message}`, 'feeder', 'info');
|
|
243
|
-
continue;
|
|
244
|
-
}
|
|
245
|
-
if (rssItemList.length === 0)
|
|
92
|
+
const feedDebug = (0, feeder_runtime_1.createFeedDebug)(config, rssItem);
|
|
93
|
+
const arg = (0, feeder_arg_1.mixinArg)(rssItem.arg || {}, config);
|
|
94
|
+
feedDebug(`[DEBUG_PROXY] feeder mixinArg result proxyAgent: ${JSON.stringify(arg.proxyAgent)}`, 'feeder', 'details');
|
|
95
|
+
const originalArg = (0, legacy_config_1.normalizeSubscriptionArg)((0, koishi_1.clone)(rssItem.arg || {}));
|
|
96
|
+
if (shouldSkipByInterval(rssItem, arg, originalArg))
|
|
246
97
|
continue;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
(0, logger_1.debug)(config, `filter:${matchKeyword}`, '', 'info');
|
|
255
|
-
(0, logger_1.debug)(config, item, 'filter rss item', 'info');
|
|
256
|
-
}
|
|
257
|
-
return !matchKeyword;
|
|
258
|
-
});
|
|
259
|
-
if (itemArray.length === 0)
|
|
98
|
+
const rssItemList = await (0, feeder_runtime_1.fetchRssItems)(ctx, config, $http, rssItem, arg, feedDebug);
|
|
99
|
+
if (rssItemList.length === 0) {
|
|
100
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
101
|
+
lastPubDate: rssItem.lastPubDate,
|
|
102
|
+
arg: originalArg,
|
|
103
|
+
lastContent: rssItem.lastContent || { itemArray: [] },
|
|
104
|
+
});
|
|
260
105
|
continue;
|
|
261
|
-
// 5. Check for Updates
|
|
262
|
-
const latestItem = itemArray[0];
|
|
263
|
-
const lastPubDate = (0, common_1.parsePubDate)(config, latestItem.pubDate);
|
|
264
|
-
(0, logger_1.debug)(config, `${rssItem.title}: Latest item date=${lastPubDate.toISOString()}, DB date=${rssItem.lastPubDate ? new Date(rssItem.lastPubDate).toISOString() : 'none'}`, 'feeder', 'details');
|
|
265
|
-
// Prepare content for deduplication
|
|
266
|
-
const currentContent = config.basic?.resendUpdataContent === 'all'
|
|
267
|
-
? itemArray.map((i) => getLastContent(i, config))
|
|
268
|
-
: [getLastContent(latestItem, config)];
|
|
269
|
-
// Reverse if needed for sending order (oldest first usually)
|
|
270
|
-
if (arg.reverse) {
|
|
271
|
-
itemArray = itemArray.reverse();
|
|
272
|
-
}
|
|
273
|
-
let rssItemArray = [];
|
|
274
|
-
if (rssItem.arg.forceLength) {
|
|
275
|
-
// Force length mode: ignore time, just take N items
|
|
276
|
-
rssItemArray = itemArray.slice(0, arg.forceLength);
|
|
277
|
-
(0, logger_1.debug)(config, `${rssItem.title}: Force length mode, taking ${rssItemArray.length} items`, 'feeder', 'details');
|
|
278
106
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
(
|
|
289
|
-
return true;
|
|
290
|
-
}
|
|
291
|
-
// Content hash check (if time is same but content changed)
|
|
292
|
-
if (config.basic?.resendUpdataContent !== 'disable') {
|
|
293
|
-
const newItemContent = getLastContent(v, config);
|
|
294
|
-
const oldItemMatch = rssItem.lastContent?.itemArray?.find((old) => (newItemContent.guid && old.guid === newItemContent.guid) ||
|
|
295
|
-
(old.link === newItemContent.link && old.title === newItemContent.title));
|
|
296
|
-
if (oldItemMatch) {
|
|
297
|
-
// If description changed, it's an update
|
|
298
|
-
const descriptionChanged = JSON.stringify(oldItemMatch.description) !== JSON.stringify(newItemContent.description);
|
|
299
|
-
if (descriptionChanged) {
|
|
300
|
-
(0, logger_1.debug)(config, `[${i}] ✓ Item is updated (content changed)`, 'feeder', 'details');
|
|
301
|
-
}
|
|
302
|
-
else {
|
|
303
|
-
(0, logger_1.debug)(config, `[${i}] ✗ Item filtered (already sent)`, 'feeder', 'details');
|
|
304
|
-
}
|
|
305
|
-
return descriptionChanged;
|
|
306
|
-
}
|
|
307
|
-
else {
|
|
308
|
-
(0, logger_1.debug)(config, `[${i}] ✗ Item filtered (no match in lastContent)`, 'feeder', 'details');
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
(0, logger_1.debug)(config, `[${i}] ✗ Item filtered (failed all checks)`, 'feeder', 'details');
|
|
312
|
-
return false;
|
|
107
|
+
const filteredItems = (0, feeder_runtime_1.filterItems)(rssItemList, arg, feedDebug);
|
|
108
|
+
if (filteredItems.length === 0) {
|
|
109
|
+
const latestItem = [...rssItemList]
|
|
110
|
+
.sort((a, b) => (0, common_1.parsePubDate)(config, b.pubDate).getTime() - (0, common_1.parsePubDate)(config, a.pubDate).getTime())[0];
|
|
111
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
112
|
+
lastPubDate: latestItem ? (0, common_1.parsePubDate)(config, latestItem.pubDate) : rssItem.lastPubDate,
|
|
113
|
+
arg: originalArg,
|
|
114
|
+
lastContent: latestItem
|
|
115
|
+
? { itemArray: [(0, feeder_runtime_1.getLastContent)(latestItem, config)] }
|
|
116
|
+
: (rssItem.lastContent || { itemArray: [] }),
|
|
313
117
|
});
|
|
314
|
-
|
|
315
|
-
if (arg.maxRssItem) {
|
|
316
|
-
rssItemArray = rssItemArray.slice(0, arg.maxRssItem);
|
|
317
|
-
}
|
|
118
|
+
continue;
|
|
318
119
|
}
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
await ctx
|
|
323
|
-
lastPubDate,
|
|
120
|
+
const { newItems, latestPubDate, currentContent } = (0, feeder_runtime_1.checkForUpdates)(config, rssItem, filteredItems, arg, feedDebug);
|
|
121
|
+
if (newItems.length === 0) {
|
|
122
|
+
feedDebug(`${rssItem.title}: No new items found after filtering`, 'feeder', 'info', { newItemCount: 0 });
|
|
123
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
124
|
+
lastPubDate: latestPubDate,
|
|
324
125
|
arg: originalArg,
|
|
325
|
-
lastContent: { itemArray: currentContent }
|
|
126
|
+
lastContent: { itemArray: currentContent },
|
|
326
127
|
});
|
|
327
128
|
continue;
|
|
328
129
|
}
|
|
329
|
-
(
|
|
330
|
-
(
|
|
331
|
-
|
|
332
|
-
const itemsToSend = [...rssItemArray].reverse();
|
|
333
|
-
// 生成所有消息
|
|
334
|
-
const messageList = (await Promise.all(itemsToSend.map(async (i) => await processor.parseRssItem(i, { ...rssItem, ...arg }, rssItem.author)))).filter(m => m); // Filter empty messages
|
|
130
|
+
feedDebug(`${rssItem.title}: Found ${newItems.length} new items`, 'feeder', 'info', { newItemCount: newItems.length });
|
|
131
|
+
feedDebug(newItems.map(i => i.title), 'feeder', 'info', { newItemCount: newItems.length });
|
|
132
|
+
const { messageList, itemsToSend } = await (0, feeder_runtime_1.generateMessages)(processor, newItems, rssItem, arg);
|
|
335
133
|
if (messageList.length === 0) {
|
|
336
|
-
(
|
|
337
|
-
|
|
338
|
-
|
|
134
|
+
feedDebug(`${rssItem.title}: Items found but parsed to empty messages`, 'feeder', 'info', { newItemCount: newItems.length });
|
|
135
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
136
|
+
lastPubDate: latestPubDate,
|
|
137
|
+
arg: originalArg,
|
|
138
|
+
lastContent: { itemArray: currentContent },
|
|
139
|
+
});
|
|
339
140
|
continue;
|
|
340
141
|
}
|
|
341
|
-
|
|
342
|
-
let message = "";
|
|
343
|
-
const shouldMerge = arg.merge === true || config.basic?.merge === '一直合并' || (config.basic?.merge === '有多条更新时合并' && messageList.length > 1);
|
|
344
|
-
// Check for video merge requirement
|
|
345
|
-
const hasVideo = config.basic?.margeVideo && messageList.some(msg => /<video/.test(msg));
|
|
346
|
-
if (shouldMerge || hasVideo) {
|
|
347
|
-
message = `<message forward><author id="${rssItem.author}"/>${messageList.map(m => `<message>${m}</message>`).join("")}</message>`;
|
|
348
|
-
}
|
|
349
|
-
else {
|
|
350
|
-
message = messageList.join("");
|
|
351
|
-
}
|
|
352
|
-
// Add mentions
|
|
353
|
-
if (rssItem.followers && rssItem.followers.length > 0) {
|
|
354
|
-
const mentions = rssItem.followers.map((id) => `<at ${id === 'all' ? 'type="all"' : `id="${id}"`}/>`).join(" ");
|
|
355
|
-
message += `<message>${mentions}</message>`;
|
|
356
|
-
}
|
|
357
|
-
// 8. 添加任务到队列(关键变更:不再直接发送)
|
|
142
|
+
const message = (0, feeder_runtime_1.buildFinalMessage)(config, messageList, rssItem, arg);
|
|
358
143
|
const taskContent = {
|
|
359
144
|
message,
|
|
360
145
|
originalItem: itemsToSend[0],
|
|
@@ -363,34 +148,48 @@ async function feeder(deps, processor) {
|
|
|
363
148
|
description: itemsToSend[0]?.description,
|
|
364
149
|
link: itemsToSend[0]?.link,
|
|
365
150
|
pubDate: (0, common_1.parsePubDate)(config, itemsToSend[0]?.pubDate),
|
|
366
|
-
imageUrl: itemsToSend[0]?.enclosure?.url
|
|
151
|
+
imageUrl: itemsToSend[0]?.enclosure?.url,
|
|
367
152
|
};
|
|
368
153
|
await queueManager.addTask({
|
|
369
154
|
subscribeId: String(rssItem.id),
|
|
370
|
-
rssId: rssItem.rssId || rssItem.title,
|
|
371
|
-
uid: itemsToSend[0]
|
|
155
|
+
rssId: String(rssItem.rssId || rssItem.title),
|
|
156
|
+
uid: buildQueueUid(itemsToSend[0], config),
|
|
372
157
|
guildId: rssItem.guildId,
|
|
373
158
|
platform: rssItem.platform,
|
|
374
|
-
content: taskContent
|
|
159
|
+
content: taskContent,
|
|
375
160
|
});
|
|
376
|
-
(
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
await ctx
|
|
380
|
-
lastPubDate,
|
|
161
|
+
feedDebug(`✓ 已添加到发送队列: ${rssItem.title}`, 'feeder', 'info', {
|
|
162
|
+
queuedItemTitle: itemsToSend[0]?.title,
|
|
163
|
+
});
|
|
164
|
+
await persistSubscriptionState(ctx, rssItem.id, {
|
|
165
|
+
lastPubDate: latestPubDate,
|
|
381
166
|
arg: originalArg,
|
|
382
|
-
lastContent: { itemArray: currentContent }
|
|
167
|
+
lastContent: { itemArray: currentContent },
|
|
383
168
|
});
|
|
384
169
|
}
|
|
385
170
|
catch (err) {
|
|
386
|
-
(0,
|
|
171
|
+
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
172
|
+
const feedContext = (0, feeder_runtime_1.buildFeedLogContext)(rssItem);
|
|
173
|
+
(0, logger_1.debug)(config, `Feeder error for ${rssItem.url}: ${normalizedError.message}`, 'feeder', 'error', feedContext);
|
|
174
|
+
(0, error_tracker_1.trackError)(normalizedError, feedContext);
|
|
387
175
|
}
|
|
388
176
|
}
|
|
389
177
|
}
|
|
390
178
|
function startFeeder(ctx, config, $http, processor, queueManager) {
|
|
391
179
|
const deps = { ctx, config, $http, queueManager };
|
|
180
|
+
const lifecycleDebug = (0, logger_1.createDebugWithContext)(config, { lifecycle: 'feeder' });
|
|
181
|
+
const queueRuntimeConfig = (0, notification_queue_retry_1.getQueueRuntimeConfig)(config);
|
|
392
182
|
// Initial run
|
|
393
|
-
feeder(deps, processor).catch(err =>
|
|
183
|
+
feeder(deps, processor).catch(err => {
|
|
184
|
+
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
185
|
+
lifecycleDebug(`Initial feeder run failed: ${normalizedError.message}`, 'feeder', 'error', {
|
|
186
|
+
operation: 'initial-feeder-run',
|
|
187
|
+
});
|
|
188
|
+
(0, error_tracker_1.trackError)(normalizedError, {
|
|
189
|
+
lifecycle: 'feeder',
|
|
190
|
+
operation: 'initial-feeder-run',
|
|
191
|
+
});
|
|
192
|
+
});
|
|
394
193
|
// 启动生产者定时器(抓取 RSS)
|
|
395
194
|
const refreshInterval = (config.basic?.refresh || 600) * 1000;
|
|
396
195
|
interval = setInterval(async () => {
|
|
@@ -402,14 +201,27 @@ function startFeeder(ctx, config, $http, processor, queueManager) {
|
|
|
402
201
|
}, refreshInterval);
|
|
403
202
|
// 启动消费者定时器(处理发送队列)
|
|
404
203
|
// 频率更高,确保消息快速发送
|
|
405
|
-
const queueProcessInterval =
|
|
204
|
+
const queueProcessInterval = queueRuntimeConfig.processIntervalSeconds * 1000;
|
|
406
205
|
queueInterval = setInterval(async () => {
|
|
407
206
|
await queueManager.processQueue();
|
|
408
207
|
}, queueProcessInterval);
|
|
409
208
|
// 立即处理一次队列(启动时)
|
|
410
|
-
queueManager.processQueue().catch(err =>
|
|
209
|
+
queueManager.processQueue().catch(err => {
|
|
210
|
+
const normalizedError = (0, error_handler_1.normalizeError)(err);
|
|
211
|
+
lifecycleDebug(`Initial queue processing failed: ${normalizedError.message}`, 'queue', 'error', {
|
|
212
|
+
operation: 'initial-queue-processing',
|
|
213
|
+
});
|
|
214
|
+
(0, error_tracker_1.trackError)(normalizedError, {
|
|
215
|
+
lifecycle: 'feeder',
|
|
216
|
+
operation: 'initial-queue-processing',
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
lifecycleDebug('Feeder started', 'feeder', 'info', {
|
|
220
|
+
refreshInterval,
|
|
221
|
+
queueProcessInterval,
|
|
222
|
+
});
|
|
411
223
|
}
|
|
412
|
-
function stopFeeder() {
|
|
224
|
+
function stopFeeder(config) {
|
|
413
225
|
if (interval) {
|
|
414
226
|
clearInterval(interval);
|
|
415
227
|
interval = null;
|
|
@@ -418,4 +230,8 @@ function stopFeeder() {
|
|
|
418
230
|
clearInterval(queueInterval);
|
|
419
231
|
queueInterval = null;
|
|
420
232
|
}
|
|
233
|
+
if (config) {
|
|
234
|
+
const lifecycleDebug = (0, logger_1.createDebugWithContext)(config, { lifecycle: 'feeder' });
|
|
235
|
+
lifecycleDebug('Feeder stopped', 'feeder', 'info');
|
|
236
|
+
}
|
|
421
237
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import * as cheerio from 'cheerio';
|
|
2
|
+
import { Context } from 'koishi';
|
|
3
|
+
import { Config, rssArg } from '../types';
|
|
4
|
+
export interface ItemProcessorRuntimeDeps {
|
|
5
|
+
ctx: Context;
|
|
6
|
+
config: Config;
|
|
7
|
+
$http: any;
|
|
8
|
+
}
|
|
9
|
+
interface RenderDescriptionOptions {
|
|
10
|
+
contentStyle?: string;
|
|
11
|
+
dividerStyle?: string;
|
|
12
|
+
logImageMode?: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* 标准化 RSS 字段内容,统一数组 / 空值 / 非字符串输入。
|
|
16
|
+
*/
|
|
17
|
+
export declare function normalizeText(value: unknown): string;
|
|
18
|
+
/**
|
|
19
|
+
* 解析并去重 HTML 中的图片资源,返回原图到最终地址的映射表。
|
|
20
|
+
*/
|
|
21
|
+
export declare function buildResolvedImageMap(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg): Promise<Record<string, string>>;
|
|
22
|
+
/**
|
|
23
|
+
* 将 HTML 中的唯一图片列表解析为消息图片串。
|
|
24
|
+
*/
|
|
25
|
+
export declare function renderImageListFromHtml(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg): Promise<string>;
|
|
26
|
+
/**
|
|
27
|
+
* 将已加载的 HTML 内容渲染为最终消息。
|
|
28
|
+
*/
|
|
29
|
+
export declare function renderLoadedHtml(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg, useXml?: boolean): Promise<string>;
|
|
30
|
+
/**
|
|
31
|
+
* 渲染模板描述正文,并在图片模式下按需注入 AI 摘要。
|
|
32
|
+
*/
|
|
33
|
+
export declare function renderTemplatedDescription(deps: ItemProcessorRuntimeDeps, item: any, arg: rssArg, description: string, options?: RenderDescriptionOptions): Promise<string>;
|
|
34
|
+
/**
|
|
35
|
+
* 提取 HTML 中的视频资源,并按配置补齐 poster 图。
|
|
36
|
+
*/
|
|
37
|
+
export declare function processVideos(deps: ItemProcessorRuntimeDeps, html: cheerio.CheerioAPI, arg: rssArg, videoList: Array<[string, string]>): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* 将视频列表格式化为最终消息片段。
|
|
40
|
+
*/
|
|
41
|
+
export declare function formatVideoList(videoList: Array<[string, string]>): string;
|
|
42
|
+
/**
|
|
43
|
+
* 统一执行图片渲染,兼容 base64 / File / assets / HTML 回退。
|
|
44
|
+
*/
|
|
45
|
+
export declare function renderImage(htmlContent: string, deps: ItemProcessorRuntimeDeps, arg: rssArg): Promise<string>;
|
|
46
|
+
export {};
|