@anyul/koishi-plugin-rss 5.0.2 → 5.0.3

package/lib/core/feeder-old.d.ts

@@ -0,0 +1,15 @@
+import { Context } from 'koishi';
+import { Config, rssArg } from '../types';
+import { RssItemProcessor } from './item-processor';
+export interface FeederDependencies {
+    ctx: Context;
+    config: Config;
+    $http: any;
+}
+export declare function findRssItem(rssList: any[], keyword: number | string): any;
+export declare function getLastContent(item: any, config: Config): any;
+export declare function formatArg(options: any, config: Config): rssArg;
+export declare function mixinArg(arg: any, config: Config): rssArg;
+export declare function feeder(deps: FeederDependencies, processor: RssItemProcessor): Promise<void>;
+export declare function startFeeder(ctx: Context, config: Config, $http: any, processor: RssItemProcessor): void;
+export declare function stopFeeder(): void;

package/lib/core/feeder-old.js

@@ -0,0 +1,403 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findRssItem = findRssItem;
+exports.getLastContent = getLastContent;
+exports.formatArg = formatArg;
+exports.mixinArg = mixinArg;
+exports.feeder = feeder;
+exports.startFeeder = startFeeder;
+exports.stopFeeder = stopFeeder;
+const koishi_1 = require("koishi");
+const logger_1 = require("../utils/logger");
+const common_1 = require("../utils/common");
+const media_1 = require("../utils/media");
+const parser_1 = require("./parser");
+const constants_1 = require("../constants");
+const message_cache_1 = require("../utils/message-cache");
+let interval = null;
+function findRssItem(rssList, keyword) {
+    let index = ((rssList.findIndex(i => i.rssId === +keyword) + 1) ||
+        (rssList.findIndex(i => i.url == keyword) + 1) ||
+        (rssList.findIndex(i => i.url.indexOf(keyword) + 1) + 1) ||
+        (rssList.findIndex(i => i.title.indexOf(keyword) + 1) + 1)) - 1;
+    return rssList[index];
+}
+function getLastContent(item, config) {
+    let arr = ['title', 'description', 'link', 'guid'];
+    let obj = Object.assign({}, ...arr.map(i => (0, koishi_1.clone)(item?.[i]) ? { [i]: item[i] } : {}));
+    return { ...obj, description: String(obj?.description).replaceAll(/\s/g, '') };
+}
+function formatArg(options, config) {
+    let { arg, template } = options;
+    let json = Object.assign({}, ...(arg?.split(',')?.map((i) => ({ [i.split(":")[0]]: i.split(":")[1] })) || []));
+    let key = ["forceLength", "reverse", "timeout", "interval", "merge", "maxRssItem", "firstLoad", "bodyWidth", "bodyPadding", "proxyAgent", "auth"];
+    let booleanKey = ['firstLoad', "reverse", 'merge'];
+    let numberKey = ['forceLength', "timeout", 'interval', 'maxRssItem', 'bodyWidth', 'bodyPadding'];
+    let falseContent = ['false', 'null', ''];
+    json = Object.assign({}, ...Object.keys(json).filter((i) => key.some((key) => key == i)).map((key) => ({ [key]: booleanKey.some((bkey) => bkey == key) ? falseContent.some((c) => c == json[key]) : numberKey.some((nkey) => nkey == key) ? (+json[key]) : json[key] })));
+    if (template && config.template) {
+        json['template'] = template;
+    }
+    // Date/Number conversions
+    if (json.interval)
+        json.interval = parseInt(json.interval) * 1000;
+    if (json.forceLength)
+        json.forceLength = parseInt(json.forceLength);
+    // Array conversions
+    if (json.filter && typeof json.filter === 'string')
+        json.filter = json.filter.split("/");
+    if (json.block && typeof json.block === 'string')
+        json.block = json.block.split("/");
+    // Proxy Argument Parsing
+    if (json.proxyAgent) {
+        if (['false', 'none', ''].includes(String(json.proxyAgent))) {
+            json.proxyAgent = { enabled: false };
+        }
+        else if (typeof json.proxyAgent === 'string') {
+            // Parse string proxy: socks5://127.0.0.1:7890
+            let protocolMatch = json.proxyAgent.match(/^(http|https|socks5)/);
+            let protocol = protocolMatch ? protocolMatch[1] : 'http';
+            let hostMatch = json.proxyAgent.match(/:\/\/([^:\/]+)/);
+            let host = hostMatch ? hostMatch[1] : '';
+            let portMatch = json.proxyAgent.match(/:(\d+)/);
+            let port = portMatch ? parseInt(portMatch[1]) : 7890;
+            let proxyAgent = { enabled: true, protocol, host, port };
+            if (json.auth) {
+                let [username, password] = json.auth.split("/");
+                proxyAgent.auth = { username, password };
+            }
+            json.proxyAgent = proxyAgent;
+        }
+    }
+    return json;
+}
+const mergeProxyAgent = (argProxy, configProxy, config) => {
+    // 打印调试信息
+    (0, logger_1.debug)(config, `合并代理配置 - argProxy: ${JSON.stringify(argProxy)}, configProxy.enabled: ${configProxy?.enabled}`, 'proxy merge debug', 'details');
+    // 1. Explicit disable in Args (必须是明确设置为 false)
+    if (argProxy?.enabled === false) {
+        (0, logger_1.debug)(config, `订阅明确禁用代理`, 'proxy merge', 'details');
+        return { enabled: false };
+    }
+    // 2. Arg 有完整的 proxy 配置 (enabled=true 且有 host) -> 使用 Arg
+    if (argProxy?.enabled === true && argProxy?.host) {
+        (0, logger_1.debug)(config, `使用订阅的代理配置`, 'proxy merge', 'details');
+        return argProxy;
+    }
+    // 3. Arg 是空对象、undefined、null，或者没有 enabled 字段 -> 使用全局配置
+    // 这是关键：如果订阅没有单独配置代理，就应该使用全局配置
+    const shouldUseConfigProxy = !argProxy || Object.keys(argProxy || {}).length === 0 || argProxy?.enabled === undefined || argProxy?.enabled === null;
+    if (shouldUseConfigProxy) {
+        if (configProxy?.enabled) {
+            const result = {
+                enabled: true,
+                protocol: configProxy.protocol,
+                host: configProxy.host,
+                port: configProxy.port,
+                auth: configProxy.auth?.enabled ? configProxy.auth : undefined
+            };
+            (0, logger_1.debug)(config, `使用全局代理: ${result.protocol}://${result.host}:${result.port}`, 'proxy merge', 'info');
+            return result;
+        }
+        else {
+            (0, logger_1.debug)(config, `全局代理未启用`, 'proxy merge', 'details');
+        }
+    }
+    // 4. Arg 的 enabled=true 但没有 host -> 尝试补充全局配置
+    if (argProxy?.enabled === true && !argProxy?.host) {
+        const result = {
+            ...configProxy,
+            ...argProxy,
+            auth: configProxy?.auth?.enabled ? configProxy.auth : undefined
+        };
+        (0, logger_1.debug)(config, `订阅代理配置不完整，补充全局配置`, 'proxy merge', 'details');
+        return result;
+    }
+    // 5. Default disabled
+    (0, logger_1.debug)(config, `代理未配置，使用默认(禁用)`, 'proxy merge', 'details');
+    return { enabled: false };
+};
+const mergeProxyAgentWithLog = (argProxy, configProxy, config) => {
+    const result = mergeProxyAgent(argProxy, configProxy, config);
+    (0, logger_1.debug)(config, `[DEBUG_PROXY] mergeProxyAgent input: arg=${JSON.stringify(argProxy)} conf=${JSON.stringify(configProxy)} output=${JSON.stringify(result)}`, 'proxy merge', 'details');
+    return result;
+};
+function mixinArg(arg, config) {
+    const mergedProxy = mergeProxyAgentWithLog(arg?.proxyAgent, config.net?.proxyAgent, config);
+    // 打印代理配置合并结果（方便调试）
+    if (mergedProxy?.enabled) {
+        (0, logger_1.debug)(config, `使用代理: ${mergedProxy.protocol}://${mergedProxy.host}:${mergedProxy.port}`, 'proxy merge', 'details');
+    }
+    else {
+        (0, logger_1.debug)(config, `代理未启用`, 'proxy merge', 'details');
+    }
+    // Flatten config into base object, prioritizing Config values
+    // We explicitly take known safe config sections
+    const baseConfig = {
+        ...config.basic,
+        // Add other flat config sections if necessary
+    };
+    const res = {
+        ...baseConfig,
+        ...arg, // Args override basic config
+        filter: [...(config.msg?.keywordFilter || []), ...(arg?.filter || [])],
+        block: [...(config.msg?.keywordBlock || []), ...(arg?.block || [])],
+        template: arg.template ?? config.basic?.defaultTemplate,
+        proxyAgent: mergedProxy
+    };
+    (0, logger_1.debug)(config, `[DEBUG_PROXY] mixinArg return: ${JSON.stringify(res.proxyAgent)}`, 'mixin', 'details');
+    return res;
+}
+async function feeder(deps, processor) {
+    const { ctx, config, $http } = deps;
+    // debug(config, "feeder run", 'debug');
+    // Use type assertion for custom table
+    const rssList = await ctx.database.get('rssOwl', {});
+    if (!rssList || rssList.length === 0)
+        return;
+    for (const rssItem of rssList) {
+        try {
+            // 1. Prepare Arguments
+            let arg = mixinArg(rssItem.arg || {}, config);
+            (0, logger_1.debug)(config, `[DEBUG_PROXY] feeder mixinArg result proxyAgent: ${JSON.stringify(arg.proxyAgent)}`, 'feeder', 'details');
+            let originalArg = (0, koishi_1.clone)(rssItem.arg || {});
+            // 2. Interval Check
+            if (rssItem.arg.interval) {
+                const now = Date.now();
+                if (arg.nextUpdataTime && arg.nextUpdataTime > now)
+                    continue;
+                // Calculate next update time
+                if (arg.nextUpdataTime) {
+                    const missed = Math.ceil((now - arg.nextUpdataTime) / arg.interval);
+                    originalArg.nextUpdataTime = arg.nextUpdataTime + (arg.interval * (missed || 1));
+                }
+                else {
+                    originalArg.nextUpdataTime = now + arg.interval;
+                }
+            }
+            // 3. Fetch RSS Data
+            // Use config.msg.rssHubUrl for quick url parsing
+            const rssHubUrl = config.msg?.rssHubUrl || 'https://hub.slarker.me';
+            let rssItemList = [];
+            try {
+                const urls = rssItem.url.split("|").map((u) => (0, common_1.parseQuickUrl)(u, rssHubUrl, constants_1.quickList));
+                const fetchPromises = urls.map((url) => (0, parser_1.getRssData)(ctx, config, $http, url, arg));
+                const results = await Promise.all(fetchPromises);
+                rssItemList = results.flat(1);
+            }
+            catch (err) {
+                (0, logger_1.debug)(config, `Fetch failed for ${rssItem.title}: ${err.message}`, 'feeder', 'info');
+                continue;
+            }
+            if (rssItemList.length === 0)
+                continue;
+            // 4. Sort and Filter
+            let itemArray = rssItemList
+                .sort((a, b) => (0, common_1.parsePubDate)(config, b.pubDate).getTime() - (0, common_1.parsePubDate)(config, a.pubDate).getTime())
+                .filter(item => {
+                // Keyword filter
+                const matchKeyword = arg.filter?.find((keyword) => new RegExp(keyword, 'im').test(item.title) || new RegExp(keyword, 'im').test(item.description));
+                if (matchKeyword) {
+                    (0, logger_1.debug)(config, `filter:${matchKeyword}`, '', 'info');
+                    (0, logger_1.debug)(config, item, 'filter rss item', 'info');
+                }
+                return !matchKeyword;
+            });
+            if (itemArray.length === 0)
+                continue;
+            // 5. Check for Updates
+            const latestItem = itemArray[0];
+            const lastPubDate = (0, common_1.parsePubDate)(config, latestItem.pubDate);
+            (0, logger_1.debug)(config, `${rssItem.title}: Latest item date=${lastPubDate.toISOString()}, DB date=${rssItem.lastPubDate ? new Date(rssItem.lastPubDate).toISOString() : 'none'}`, 'feeder', 'details');
+            // Prepare content for deduplication
+            const currentContent = config.basic?.resendUpdataContent === 'all'
+                ? itemArray.map((i) => getLastContent(i, config))
+                : [getLastContent(latestItem, config)];
+            // Reverse if needed for sending order (oldest first usually)
+            if (arg.reverse) {
+                itemArray = itemArray.reverse();
+            }
+            let rssItemArray = [];
+            if (rssItem.arg.forceLength) {
+                // Force length mode: ignore time, just take N items
+                rssItemArray = itemArray.slice(0, arg.forceLength);
+                (0, logger_1.debug)(config, `${rssItem.title}: Force length mode, taking ${rssItemArray.length} items`, 'feeder', 'details');
+            }
+            else {
+                // Standard mode: Time & Content check
+                (0, logger_1.debug)(config, `${rssItem.title}: Checking ${itemArray.length} items for updates`, 'feeder', 'details');
+                rssItemArray = itemArray.filter((v, i) => {
+                    const currentItemTime = (0, common_1.parsePubDate)(config, v.pubDate).getTime();
+                    const lastTime = rssItem.lastPubDate ? (0, common_1.parsePubDate)(config, rssItem.lastPubDate).getTime() : 0;
+                    (0, logger_1.debug)(config, `[${i}] ${v.title?.substring(0, 30)}: time=${new Date(currentItemTime).toISOString()} > last=${new Date(lastTime).toISOString()} ? ${currentItemTime > lastTime}`, 'feeder', 'details');
+                    // Strict time check
+                    if (currentItemTime > lastTime) {
+                        (0, logger_1.debug)(config, `[${i}] ✓ Item is new (time check)`, 'feeder', 'details');
+                        return true;
+                    }
+                    // Content hash check (if time is same but content changed)
+                    if (config.basic?.resendUpdataContent !== 'disable') {
+                        const newItemContent = getLastContent(v, config);
+                        const oldItemMatch = rssItem.lastContent?.itemArray?.find((old) => (newItemContent.guid && old.guid === newItemContent.guid) ||
+                            (old.link === newItemContent.link && old.title === newItemContent.title));
+                        if (oldItemMatch) {
+                            // If description changed, it's an update
+                            const descriptionChanged = JSON.stringify(oldItemMatch.description) !== JSON.stringify(newItemContent.description);
+                            if (descriptionChanged) {
+                                (0, logger_1.debug)(config, `[${i}] ✓ Item is updated (content changed)`, 'feeder', 'details');
+                            }
+                            else {
+                                (0, logger_1.debug)(config, `[${i}] ✗ Item filtered (already sent)`, 'feeder', 'details');
+                            }
+                            return descriptionChanged;
+                        }
+                        else {
+                            (0, logger_1.debug)(config, `[${i}] ✗ Item filtered (no match in lastContent)`, 'feeder', 'details');
+                        }
+                    }
+                    (0, logger_1.debug)(config, `[${i}] ✗ Item filtered (failed all checks)`, 'feeder', 'details');
+                    return false;
+                });
+                // Apply Max Item Limit
+                if (arg.maxRssItem) {
+                    rssItemArray = rssItemArray.slice(0, arg.maxRssItem);
+                }
+            }
+            if (rssItemArray.length === 0) {
+                (0, logger_1.debug)(config, `${rssItem.title}: No new items found after filtering`, 'feeder', 'info');
+                // No new items, but we should still update 'lastContent' to latest state to prevent future drifts
+                await ctx.database.set('rssOwl', { id: rssItem.id }, {
+                    lastPubDate,
+                    arg: originalArg,
+                    lastContent: { itemArray: currentContent }
+                });
+                continue;
+            }
+            (0, logger_1.debug)(config, `${rssItem.title}: Found ${rssItemArray.length} new items`, 'feeder', 'info');
+            (0, logger_1.debug)(config, rssItemArray.map(i => i.title), '', 'info');
+            // 6. Process Items (Generate Messages)
+            const itemsToSend = [...rssItemArray].reverse();
+            const messageList = (await Promise.all(itemsToSend.map(async (i) => await processor.parseRssItem(i, { ...rssItem, ...arg }, rssItem.author)))).filter(m => m); // Filter empty messages
+            if (messageList.length === 0) {
+                (0, logger_1.debug)(config, `${rssItem.title}: Items found but parsed to empty messages`, 'feeder', 'info');
+                // Items found but parsed to empty (e.g. filtered by video mode)
+                await ctx.database.set('rssOwl', { id: rssItem.id }, { lastPubDate, arg: originalArg, lastContent: { itemArray: currentContent } });
+                continue;
+            }
+            // 7. Construct Final Message
+            let message = "";
+            const shouldMerge = arg.merge === true || config.basic?.merge === '一直合并' || (config.basic?.merge === '有多条更新时合并' && messageList.length > 1);
+            // Check for video merge requirement
+            const hasVideo = config.basic?.margeVideo && messageList.some(msg => /<video/.test(msg));
+            if (shouldMerge || hasVideo) {
+                message = `<message forward><author id="${rssItem.author}"/>${messageList.map(m => `<message>${m}</message>`).join("")}</message>`;
+            }
+            else {
+                message = messageList.join("");
+            }
+            // Add mentions
+            if (rssItem.followers && rssItem.followers.length > 0) {
+                const mentions = rssItem.followers.map((id) => `<at ${id === 'all' ? 'type="all"' : `id="${id}"`}/>`).join(" ");
+                message += `<message>${mentions}</message>`;
+            }
+            // 8. Send Broadcast
+            try {
+                (0, logger_1.debug)(config, `Sending update for ${rssItem.title} to ${rssItem.platform}:${rssItem.guildId}`, 'feeder', 'details');
+                // Koishi broadcast 会自动查找可用的 bot，无需手动检查
+                // author 字段兼容用户ID和bot selfId两种格式
+                // 发送消息
+                try {
+                    await ctx.broadcast([`${rssItem.platform}:${rssItem.guildId}`], message);
+                    (0, logger_1.debug)(config, `更新成功:${rssItem.title}`, '', 'info');
+                }
+                catch (sendError) {
+                    // OneBot retcode 1200: 不支持的消息格式（通常是视频）
+                    if (sendError.code?.toString?.() === '1200' || sendError.message?.includes('1200')) {
+                        (0, logger_1.debug)(config, `消息格式不被支持，尝试清理视频元素后重试: ${rssItem.title}`, 'feeder', 'info');
+                        // 移除 video 元素，保留视频链接
+                        const fallbackMessage = message
+                            .replace(/<video[^>]*>.*?<\/video>/gis, (match) => {
+                            // 提取视频 URL
+                            const srcMatch = match.match(/src=["']([^"']+)["']/);
+                            if (srcMatch) {
+                                return `\n🎬 视频: ${srcMatch[1]}\n`;
+                            }
+                            return '\n[视频不支持]\n';
+                        });
+                        try {
+                            await ctx.broadcast([`${rssItem.platform}:${rssItem.guildId}`], fallbackMessage);
+                            (0, logger_1.debug)(config, `降级发送成功:${rssItem.title}`, '', 'info');
+                        }
+                        catch (retryError) {
+                            (0, logger_1.debug)(config, `降级发送也失败: ${retryError.message}`, 'feeder', 'error');
+                            throw retryError;
+                        }
+                    }
+                    else {
+                        throw sendError;
+                    }
+                }
+                // 缓存最终发送的消息
+                if (config.cache?.enabled && messageList.length > 0) {
+                    const cache = (0, message_cache_1.getMessageCache)();
+                    if (cache) {
+                        // 缓存每条消息的最终形式
+                        for (let i = 0; i < itemsToSend.length && i < messageList.length; i++) {
+                            const item = itemsToSend[i];
+                            const finalMsg = messageList[i];
+                            try {
+                                await cache.addMessage({
+                                    rssId: rssItem.rssId.toString(),
+                                    guildId: rssItem.guildId,
+                                    platform: rssItem.platform,
+                                    title: item.title || '',
+                                    content: item.description || '',
+                                    link: item.link || '',
+                                    pubDate: (0, common_1.parsePubDate)(config, item.pubDate),
+                                    imageUrl: item.enclosure?.url || '',
+                                    videoUrl: '',
+                                    finalMessage: finalMsg // 缓存最终发送的消息
+                                });
+                            }
+                            catch (err) {
+                                (0, logger_1.debug)(config, `缓存消息失败: ${err.message}`, 'cache', 'info');
+                            }
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                (0, logger_1.debug)(config, `RSS推送失败 [${rssItem.title}]: ${err.message}`, 'feeder', 'error');
+                console.error(`RSS推送失败 [${rssItem.title}]: ${err.message}`);
+                // 即使发送失败，也要更新数据库状态，避免无限重试
+            }
+            // 9. Update Database State
+            await ctx.database.set('rssOwl', { id: rssItem.id }, {
+                lastPubDate,
+                arg: originalArg,
+                lastContent: { itemArray: currentContent }
+            });
+        }
+        catch (err) {
+            (0, logger_1.debug)(config, `Feeder error for ${rssItem.url}: ${err.message}`, 'feeder', 'error');
+        }
+    }
+}
+function startFeeder(ctx, config, $http, processor) {
+    const deps = { ctx, config, $http };
+    // Initial run
+    feeder(deps, processor).catch(err => console.error("Initial feeder run failed:", err));
+    const refreshInterval = (config.basic?.refresh || 600) * 1000;
+    interval = setInterval(async () => {
+        if (config.basic?.imageMode === 'File') {
+            await (0, media_1.delCache)(config);
+        }
+        await feeder(deps, processor);
+    }, refreshInterval);
+}
+function stopFeeder() {
+    if (interval) {
+        clearInterval(interval);
+        interval = null;
+    }
+}

package/lib/core/feeder.d.ts

@@ -1,15 +1,20 @@
 import { Context } from 'koishi';
 import { Config, rssArg } from '../types';
 import { RssItemProcessor } from './item-processor';
+import { NotificationQueueManager } from './notification-queue';
 export interface FeederDependencies {
     ctx: Context;
     config: Config;
     $http: any;
+    queueManager: NotificationQueueManager;
 }
 export declare function findRssItem(rssList: any[], keyword: number | string): any;
 export declare function getLastContent(item: any, config: Config): any;
 export declare function formatArg(options: any, config: Config): rssArg;
 export declare function mixinArg(arg: any, config: Config): rssArg;
+/**
+ * 生产者：抓取 RSS，发现新消息，存入队列
+ */
 export declare function feeder(deps: FeederDependencies, processor: RssItemProcessor): Promise<void>;
-export declare function startFeeder(ctx: Context, config: Config, $http: any, processor: RssItemProcessor): void;
+export declare function startFeeder(ctx: Context, config: Config, $http: any, processor: RssItemProcessor, queueManager: NotificationQueueManager): void;
 export declare function stopFeeder(): void;