ns-rss-spider 0.0.18 → 0.0.20

package/dist/cjs/parse.d.ts

@@ -1,7 +1,6 @@
 import { ServerInfo } from "./upload";
 export declare function parseRss(name: string, feed: string, server?: ServerInfo): Promise<import("./types").Article[] | {
     storePath: any;
-    source?: string | undefined;
     type: "simple" | "rich";
     guid: string;
     title: string;
@@ -9,5 +8,6 @@ export declare function parseRss(name: string, feed: string, server?: ServerInfo
     description: string;
     pubDate: Date;
     author?: string | undefined;
+    source?: string | undefined;
     categories?: string | undefined;
 }[]>;

package/dist/cjs/parseContent/index.js

@@ -49,7 +49,7 @@ async function parseContent(item, strategy) {
     console.log(import_zx.chalk.green("正在拉取文章内容"), item.link);
     const article = await (0, import_getArticleHtml.getArticleHtml)(item.link, strategy.fetcher);
     $ = cheerio.load(article);
-    element = ((_a = strategy.getContentHtmlFromArticle) == null ? void 0 : _a.call(strategy, $)) || $.root();
+    element = ((_a = strategy.getContentElementFromArticle) == null ? void 0 : _a.call(strategy, $)) || $.root();
   }
   console.log(import_zx.chalk.green("正在预处理 html"));
   (0, import_stripeHtml.stripeHtml)($, element);

package/dist/cjs/parseContent/parseContent.js

@@ -39,6 +39,7 @@ var import_html_entities = require("html-entities");
 var import_generateSsrContent = require("./generateSsrContent");
 var import_getBasicFromItem = require("./getBasicFromItem");
 async function parseContent($, $element, item, strategy) {
+  var _a;
   const srcs = [];
   const images = [];
   (0, import_utils.walk_the_DOM)($, $element, (node) => {
@@ -54,6 +55,13 @@ async function parseContent($, $element, item, strategy) {
     }
   });
   for (let item2 of srcs) {
+    if (strategy == null ? void 0 : strategy.ignoreProbeImage) {
+      images.push({
+        url: item2.src,
+        title: item2.title
+      });
+      continue;
+    }
     console.log(import_zx.chalk.green("正在解析图片", item2.src));
     const result = await (0, import_probe_image_size.default)(item2.src, {
       rejectUnauthorized: false,
@@ -93,7 +101,8 @@ async function parseContent($, $element, item, strategy) {
       }
     }
   });
-  const content = (0, import_html_entities.decode)($element.html().trim());
+  const elCont = ((_a = strategy == null ? void 0 : strategy.getContentFromHtml) == null ? void 0 : _a.call(strategy, $, $element)) || $element.html() || "";
+  const content = (0, import_html_entities.decode)(elCont.trim());
   return {
     type: "rich",
     ...(0, import_getBasicFromItem.getBasicFromItem)(item),

package/dist/cjs/parsers/cnbeta.js

@@ -25,18 +25,25 @@ module.exports = __toCommonJS(cnbeta_exports);
 var cnbeta = {
   parse: true,
   fetcher: "http",
-  getContentHtmlFromArticle: ($) => {
+  getContentElementFromArticle: ($) => {
     const el = $(".article-content");
-    el.remove(".tac");
     return el;
   },
+  getContentFromHtml: ($, $el) => {
+    const desc = $(".cnbeta-article-body .article-summary");
+    if (desc.length) {
+      $(desc).find(".topic").remove();
+      $(desc).find("p").removeAttr("class").addClass("introduction");
+    }
+    return (desc.html() || "") + ($el.html() || "");
+  },
   getExtraItems: async ($, rich) => {
     return {
       description: rich.description.replace(`
- 阅读全文`, ""),
-      pageFrom: $(".cnbeta-article .title .source a").text().toLowerCase()
+ 阅读全文`, "")
     };
-  }
+  },
+  ignoreProbeImage: true
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parsers/ifanr.js

@@ -23,12 +23,7 @@ __export(ifanr_exports, {
 });
 module.exports = __toCommonJS(ifanr_exports);
 var ifanr = {
-  parse: true,
-  fetcher: "http",
-  getContentHtmlFromArticle: ($) => {
-    const el = $("#article-content-wrapper article");
-    return el;
-  }
+  parse: true
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parsers/ithome.js

@@ -26,7 +26,7 @@ var import_utils = require("../parseContent/utils");
 var ithome = {
   parse: true,
   fetcher: "http",
-  getContentHtmlFromArticle: ($) => {
+  getContentElementFromArticle: ($) => {
     const el = $("#paragraph");
     $(el).find("img").each((_, img) => {
       const original = $(img).attr("data-original");

package/dist/cjs/types.d.ts

@@ -7,8 +7,10 @@ export type RssItem = Item & {
 export interface ParseStrategy {
     parse: boolean;
     fetcher?: 'http' | 'playwright';
-    getContentHtmlFromArticle?: (aritcle: CheerioAPI) => Cheerio<AnyNode>;
-    getExtraItems?: (html: CheerioAPI, current: RichArticle, item: Item) => Promise<Record<string, any>>;
+    getContentElementFromArticle?: (aritcle: CheerioAPI) => Cheerio<AnyNode>;
+    getContentFromHtml?: ($: CheerioAPI, node: Cheerio<AnyNode>) => string;
+    getExtraItems?: ($: CheerioAPI, current: RichArticle, item: Item) => Promise<Record<string, any>>;
+    ignoreProbeImage?: boolean;
 }
 export interface SimpleArticle {
     type: 'simple';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",