@anyul/koishi-plugin-rss 5.2.3 → 5.2.41

package/lib/core/ai-client.js

@@ -0,0 +1,65 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requestAiText = requestAiText;
+exports.callAiApi = callAiApi;
+const axios_1 = __importDefault(require("axios"));
+const logger_1 = require("../utils/logger");
+const proxy_1 = require("../utils/proxy");
+async function requestAiText(config, prompt, options = {}) {
+    const requestConfig = {
+        headers: {
+            'Authorization': `Bearer ${config.ai.apiKey}`,
+            'Content-Type': 'application/json'
+        },
+        timeout: options.timeout ?? config.ai?.timeout,
+        ...(0, proxy_1.buildAxiosProxyConfig)(config)
+    };
+    const response = await axios_1.default.post(`${config.ai.baseUrl.replace(/\/+$/, '')}/chat/completions`, {
+        model: config.ai.model,
+        messages: [{ role: 'user', content: prompt }],
+        temperature: options.temperature ?? 0.7
+    }, requestConfig);
+    const summary = response.data?.choices?.[0]?.message?.content?.trim();
+    if (!summary) {
+        throw new Error('AI 返回空结果');
+    }
+    return summary;
+}
+async function callAiApi(config, prompt, context) {
+    const maxRetries = 2;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            (0, logger_1.debug)(config, `AI 请求尝试 ${attempt + 1}/${maxRetries + 1}: ${context}`, 'AI', 'info');
+            const summary = await requestAiText(config, prompt);
+            (0, logger_1.debug)(config, `AI 摘要生成成功: ${summary.substring(0, 20)}...`, 'AI', 'details');
+            return {
+                success: true,
+                summary,
+                cached: false
+            };
+        }
+        catch (error) {
+            const isLastAttempt = attempt === maxRetries;
+            (0, logger_1.debug)(config, `AI 请求失败 (尝试 ${attempt + 1}/${maxRetries + 1}): ${error.message}`, 'AI', isLastAttempt ? 'error' : 'info');
+            if (isLastAttempt) {
+                return {
+                    success: false,
+                    summary: '',
+                    cached: false,
+                    error: error.message
+                };
+            }
+            const waitTime = Math.pow(2, attempt) * 1000;
+            await new Promise(resolve => setTimeout(resolve, waitTime));
+        }
+    }
+    return {
+        success: false,
+        summary: '',
+        cached: false,
+        error: '未知错误'
+    };
+}

package/lib/core/ai-selector.d.ts

@@ -0,0 +1,2 @@
+import { Config } from '../types';
+export declare function generateSelectorByAI(config: Config, url: string, instruction: string, html: string): Promise<string>;

package/lib/core/ai-selector.js

@@ -0,0 +1,80 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateSelectorByAI = generateSelectorByAI;
+const cheerio = __importStar(require("cheerio"));
+const logger_1 = require("../utils/logger");
+const ai_client_1 = require("./ai-client");
+async function generateSelectorByAI(config, url, instruction, html) {
+    if (!config.ai?.enabled || !config.ai?.apiKey) {
+        throw new Error('需在配置中开启 AI 功能并填写 API Key');
+    }
+    const $ = cheerio.load(html);
+    $('script, style, svg, path, link, meta, noscript').remove();
+    $('*').contents().each((_, element) => {
+        if (element.type === 'comment') {
+            $(element).remove();
+        }
+    });
+    const cleanHtml = $('body').html()?.replace(/\s+/g, ' ').trim().substring(0, 15000) || '';
+    const prompt = `
+    作为一名爬虫专家，请根据提供的 HTML 代码片段，为一个网页监控工具生成一个 CSS Selector。
+
+    目标网页：${url}
+    用户需求：${instruction}
+
+    要求：
+    1. 只返回 CSS Selector 字符串，不要包含任何解释、Markdown 标记或代码块符号。
+    2. Selector 必须尽可能精确，通常用于提取列表中的一项或多项。
+    3. 如果是列表，请确保 Selector 能选中列表项的容器。
+
+    HTML片段：
+    ${cleanHtml}
+    `;
+    try {
+        (0, logger_1.debug)(config, `正在请求 AI 生成选择器: ${instruction}`, 'AI-Selector', 'info');
+        let selector = await (0, ai_client_1.requestAiText)(config, prompt, {
+            temperature: 0.1,
+            timeout: 60000
+        });
+        selector = selector.replace(/`/g, '').replace(/^css/i, '').trim();
+        (0, logger_1.debug)(config, `AI 生成的选择器: ${selector}`, 'AI-Selector', 'info');
+        return selector || '';
+    }
+    catch (error) {
+        (0, logger_1.debug)(config, `AI 生成选择器失败: ${error.message}`, 'AI-Selector', 'error');
+        throw error;
+    }
+}

package/lib/core/ai-summary.d.ts

@@ -0,0 +1,10 @@
+import { Config } from '../types';
+export declare function getAiSummary(config: Config, title: string, contentHtml: string): Promise<string>;
+export declare function getBatchAiSummary(config: Config, items: Array<{
+    title: string;
+    content: string;
+}>): Promise<string>;
+export declare function getSmartAiSummary(config: Config, items: Array<{
+    title: string;
+    content: string;
+}>): Promise<string>;

package/lib/core/ai-summary.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAiSummary = getAiSummary;
+exports.getBatchAiSummary = getBatchAiSummary;
+exports.getSmartAiSummary = getSmartAiSummary;
+const logger_1 = require("../utils/logger");
+const ai_cache_1 = require("./ai-cache");
+const ai_client_1 = require("./ai-client");
+const ai_utils_1 = require("./ai-utils");
+function shouldSkipAiSummary(config) {
+    return !config.ai?.enabled || !config.ai?.apiKey;
+}
+function getSummaryCache(config) {
+    return (0, ai_cache_1.getOrInitAiCache)(undefined, config.security?.maxCacheSize);
+}
+async function getAiSummary(config, title, contentHtml) {
+    if (shouldSkipAiSummary(config))
+        return '';
+    const cache = getSummaryCache(config);
+    const plainText = (0, ai_utils_1.cleanHtmlContent)(contentHtml, config.ai.maxInputLength);
+    if (!plainText || plainText.length < 50)
+        return '';
+    const cachedSummary = cache.get(title, plainText);
+    if (cachedSummary) {
+        (0, logger_1.debug)(config, `使用缓存的 AI 摘要: ${title}`, 'AI-Cache', 'details');
+        return cachedSummary;
+    }
+    let prompt = (0, ai_utils_1.buildSingleSummaryPrompt)(config.ai.prompt, title, plainText);
+    const searchQuery = title || plainText.substring(0, 100);
+    prompt = await (0, ai_utils_1.enhancePromptWithSearch)(config, prompt, searchQuery);
+    const result = await (0, ai_client_1.callAiApi)(config, prompt, `单条摘要: ${title}`);
+    if (result.success && result.summary) {
+        cache.set(title, plainText, result.summary);
+    }
+    return result.summary;
+}
+async function getBatchAiSummary(config, items) {
+    if (shouldSkipAiSummary(config) || items.length === 0)
+        return '';
+    if (items.length === 1) {
+        return getAiSummary(config, items[0].title, items[0].content);
+    }
+    (0, logger_1.debug)(config, `批量生成 AI 摘要: ${items.length} 条内容`, 'AI-Batch', 'info');
+    try {
+        const cleanedItems = (0, ai_utils_1.cleanSummaryItems)(items, config.ai.maxInputLength);
+        if (cleanedItems.length === 0) {
+            (0, logger_1.debug)(config, '所有内容都太短，无法生成批量摘要', 'AI-Batch', 'info');
+            return '';
+        }
+        let prompt = (0, ai_utils_1.buildBatchSummaryPrompt)(cleanedItems);
+        prompt = await (0, ai_utils_1.enhancePromptWithSearch)(config, prompt, cleanedItems[0].title, '批量摘要 - ');
+        const result = await (0, ai_client_1.callAiApi)(config, prompt, `批量摘要: ${cleanedItems.length}条`);
+        if (result.success && result.summary) {
+            (0, logger_1.debug)(config, `批量摘要生成成功: ${result.summary.substring(0, 50)}...`, 'AI-Batch', 'details');
+        }
+        return result.summary;
+    }
+    catch (error) {
+        (0, logger_1.debug)(config, `批量摘要生成失败: ${error.message}`, 'AI-Batch', 'error');
+        return '';
+    }
+}
+async function getSmartAiSummary(config, items) {
+    if (shouldSkipAiSummary(config) || items.length === 0) {
+        return '';
+    }
+    const threshold = 3;
+    if (items.length > threshold) {
+        return getBatchAiSummary(config, items);
+    }
+    const summaries = await Promise.all(items.map(item => getAiSummary(config, item.title, item.content)));
+    return summaries.filter(summary => summary).join('\n\n');
+}

package/lib/core/ai-utils.d.ts

@@ -0,0 +1,10 @@
+import { Config } from '../types';
+export interface AiSummaryItem {
+    title: string;
+    content: string;
+}
+export declare function cleanHtmlContent(contentHtml: string, maxLength: number): string;
+export declare function buildSingleSummaryPrompt(promptTemplate: string, title: string, plainText: string): string;
+export declare function buildBatchSummaryPrompt(items: AiSummaryItem[]): string;
+export declare function cleanSummaryItems(items: AiSummaryItem[], maxInputLength: number): AiSummaryItem[];
+export declare function enhancePromptWithSearch(config: Config, prompt: string, searchQuery: string, logPrefix?: string): Promise<string>;

package/lib/core/ai-utils.js

@@ -0,0 +1,104 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cleanHtmlContent = cleanHtmlContent;
+exports.buildSingleSummaryPrompt = buildSingleSummaryPrompt;
+exports.buildBatchSummaryPrompt = buildBatchSummaryPrompt;
+exports.cleanSummaryItems = cleanSummaryItems;
+exports.enhancePromptWithSearch = enhancePromptWithSearch;
+const cheerio = __importStar(require("cheerio"));
+const config_1 = require("../config");
+const logger_1 = require("../utils/logger");
+const search_format_1 = require("./search-format");
+const search_service_1 = require("./search-service");
+function cleanHtmlContent(contentHtml, maxLength) {
+    const $ = cheerio.load(contentHtml || '');
+    $('script').remove();
+    $('style').remove();
+    $('img').remove();
+    $('video').remove();
+    let plainText = $.text().replace(/\s+/g, ' ').trim();
+    if (plainText.length > maxLength) {
+        plainText = plainText.substring(0, maxLength) + '...';
+    }
+    return plainText;
+}
+function buildSingleSummaryPrompt(promptTemplate, title, plainText) {
+    return promptTemplate
+        .replace('{{title}}', title || '')
+        .replace('{{content}}', plainText);
+}
+function buildBatchSummaryPrompt(items) {
+    return `请简要总结以下 ${items.length} 条新闻/文章的核心内容，要求：
+1. 语言简洁流畅，每条总结不超过30字
+2. 按顺序总结，使用数字编号
+3. 突出重点信息
+
+${items.map((item, index) => `
+${index + 1}. 标题：${item.title}
+   内容：${item.content}
+`).join('\n')}
+
+总结：`;
+}
+function cleanSummaryItems(items, maxInputLength) {
+    return items
+        .map(item => ({
+        title: item.title,
+        content: cleanHtmlContent(item.content, maxInputLength / items.length)
+    }))
+        .filter(item => item.content.length >= 50);
+}
+async function enhancePromptWithSearch(config, prompt, searchQuery, logPrefix = '') {
+    if (!config.search?.enabled) {
+        return prompt;
+    }
+    try {
+        const normalizedSearchConfig = (0, config_1.normalizeSearchConfig)(config.search);
+        (0, logger_1.debug)(config, `${logPrefix}正在联网搜索: ${searchQuery}`, 'AI-Search', 'info');
+        const searchResults = await (0, search_service_1.webSearch)(config, searchQuery, normalizedSearchConfig);
+        if (searchResults.success && searchResults.results.length > 0) {
+            (0, logger_1.debug)(config, `${logPrefix}联网搜索成功，找到 ${searchResults.results.length} 条结果`, 'AI-Search', 'details');
+            return (0, search_format_1.buildPromptWithSearchContext)(prompt, searchResults, searchQuery);
+        }
+        if (searchResults.error) {
+            (0, logger_1.debug)(config, `${logPrefix}联网搜索失败: ${searchResults.error}`, 'AI-Search', 'info');
+        }
+    }
+    catch (error) {
+        (0, logger_1.debug)(config, `${logPrefix}联网搜索异常: ${error.message}`, 'AI-Search', 'error');
+    }
+    return prompt;
+}

package/lib/core/ai.d.ts

@@ -1,91 +1,3 @@
-import { Config } from '../types';
-/**
- * AI 摘要缓存管理器
- */
-export declare class AiSummaryCache {
-    private cache;
-    private ttl;
-    private maxSize;
-    private accessOrder;
-    constructor(ttl?: number, maxSize?: number);
-    /**
-     * 触发 LRU 淘汰，移除最久未访问的条目
-     */
-    private evictIfNeeded;
-    /**
-     * 更新访问顺序（LRU）
-     */
-    private updateAccessOrder;
-    /**
-     * 从访问顺序中移除指定键
-     */
-    private removeAccessOrder;
-    /**
-     * 生成缓存键（基于内容的哈希）
-     */
-    private generateKey;
-    /**
-     * 获取缓存
-     */
-    get(title: string, content: string): string | null;
-    /**
-     * 设置缓存
-     */
-    set(title: string, content: string, summary: string): void;
-    /**
-     * 清除过期缓存
-     */
-    cleanExpired(): void;
-    /**
-     * 清空所有缓存
-     */
-    clear(): void;
-    /**
-     * 获取缓存统计
-     */
-    getStats(): {
-        size: number;
-        keys: string[];
-    };
-}
-/**
- * 初始化 AI 摘要缓存
- */
-export declare function initAiCache(ttl?: number, maxSize?: number): void;
-/**
- * 生成单条 AI 摘要（带缓存和降级）
- */
-export declare function getAiSummary(config: Config, title: string, contentHtml: string): Promise<string>;
-/**
- * 批量生成 AI 摘要（多条更新合并为一条）
- */
-export declare function getBatchAiSummary(config: Config, items: Array<{
-    title: string;
-    content: string;
-}>): Promise<string>;
-/**
- * 智能摘要：根据内容数量自动选择单条或批量摘要
- */
-export declare function getSmartAiSummary(config: Config, items: Array<{
-    title: string;
-    content: string;
-}>): Promise<string>;
-/**
- * 清除过期缓存
- */
-export declare function cleanExpiredCache(): void;
-/**
- * 清空所有缓存
- */
-export declare function clearAiCache(): void;
-/**
- * 获取缓存统计信息
- */
-export declare function getAiCacheStats(): {
-    size: number;
-    keys: string[];
-} | null;
-/**
- * AI 智能生成 CSS 选择器（保持原有功能）
- */
-export declare function generateSelectorByAI(config: Config, url: string, instruction: string, html: string): Promise<string>;
+export { AiSummaryCache, cleanExpiredCache, clearAiCache, getAiCacheStats, initAiCache } from './ai-cache';
+export { generateSelectorByAI } from './ai-selector';
+export { getAiSummary, getBatchAiSummary, getSmartAiSummary } from './ai-summary';