ns-rss-spider 0.0.29 → 0.0.30

package/dist/cjs/index.d.ts

@@ -1,3 +1,4 @@
 export { parseRss } from "./parse";
 export { strategies } from "./strategy";
 export { parseContent } from "./parseContent";
+export { getWebContent } from './utils/browser';

package/dist/cjs/index.js

@@ -19,6 +19,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
+  getWebContent: () => import_browser.getWebContent,
   parseContent: () => import_parseContent.parseContent,
   parseRss: () => import_parse.parseRss,
   strategies: () => import_strategy.strategies
@@ -27,8 +28,10 @@ module.exports = __toCommonJS(src_exports);
 var import_parse = require("./parse");
 var import_strategy = require("./strategy");
 var import_parseContent = require("./parseContent");
+var import_browser = require("./utils/browser");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  getWebContent,
   parseContent,
   parseRss,
   strategies

package/dist/cjs/parsers/36kr.js

@@ -22,8 +22,36 @@ __export(kr_exports, {
   _36kr: () => _36kr
 });
 module.exports = __toCommonJS(kr_exports);
+var import_zx = require("zx");
+var import_cheerio = require("cheerio");
+var import_constants = require("../utils/constants");
+var import_browser = require("../utils/browser");
 var _36kr = {
-  parse: true
+  parse: true,
+  getThumbs: async (items) => {
+    const url = "https://m.36kr.com/";
+    const html = await (0, import_browser.getWebContent)(url, {
+      userAgent: import_constants.iosUA
+    }).catch((e) => {
+      console.error(import_zx.chalk.red(`获取 ${url} 错误`), e);
+    });
+    if (!html)
+      return void 0;
+    const $ = (0, import_cheerio.load)(html, null, false);
+    const kv = {};
+    $(".home-flow > .flow-list .article-item a").each((_, $a) => {
+      var _a, _b;
+      const thumb = $($a).find("img").attr("src");
+      const href = $($a).attr("href") || "";
+      const reg = /\/p\/([\d]+)/;
+      const articleId = (_a = reg.exec(href)) == null ? void 0 : _a[1];
+      const guid = (_b = items.find((a) => (a.guid || a.link || "").includes(`/p/${articleId}`))) == null ? void 0 : _b.guid;
+      if (guid && thumb) {
+        kv[guid] = thumb;
+      }
+    });
+    return kv;
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parsers/cnbeta.js

@@ -35,6 +35,7 @@ module.exports = __toCommonJS(cnbeta_exports);
 var import_axios = __toESM(require("axios"));
 var import_zx = require("zx");
 var import_cheerio = require("cheerio");
+var import_constants = require("../utils/constants");
 var cnbeta = {
   parse: true,
   fetcher: "http",
@@ -59,7 +60,10 @@ var cnbeta = {
   ignoreProbeImage: true,
   getThumbs: async (items) => {
     const res = await import_axios.default.get("https://m.cnbeta.com.tw/", {
-      responseType: "text"
+      responseType: "text",
+      headers: {
+        "User-Agent": import_constants.iosUA
+      }
     }).catch((e) => {
       console.error(import_zx.chalk.red("获取 m.cnbeta.com.tw 错误"), e);
     });

package/dist/cjs/parsers/ithome.js

@@ -36,6 +36,7 @@ var import_utils = require("../parseContent/utils");
 var import_axios = __toESM(require("axios"));
 var import_zx = require("zx");
 var import_cheerio = require("cheerio");
+var import_constants = require("../utils/constants");
 var ithome = {
   parse: true,
   fetcher: "http",
@@ -64,7 +65,10 @@ var ithome = {
   },
   getThumbs: async (articles) => {
     const res = await import_axios.default.get("https://m.ithome.com/", {
-      responseType: "text"
+      responseType: "text",
+      headers: {
+        "User-Agent": import_constants.iosUA
+      }
     }).catch((e) => {
       console.error(import_zx.chalk.red("获取 https://m.ithome.com/ 错误"), e);
     });

package/dist/cjs/utils/browser.d.ts

@@ -0,0 +1,3 @@
+export declare function getWebContent(url: string, options?: {
+    userAgent?: string;
+}): Promise<string>;

package/dist/cjs/utils/browser.js

@@ -0,0 +1,46 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/browser.ts
+var browser_exports = {};
+__export(browser_exports, {
+  getWebContent: () => getWebContent
+});
+module.exports = __toCommonJS(browser_exports);
+var import_playwright = require("playwright");
+async function getWebContent(url, options) {
+  const browser = await import_playwright.chromium.launch();
+  const page = await browser.newPage({
+    userAgent: options == null ? void 0 : options.userAgent
+  });
+  let resolve;
+  const waitForLoadPromise = new Promise((r) => {
+    resolve = r;
+  });
+  page.on("load", (page2) => {
+    page2.locator("html").innerHTML().then((h) => resolve == null ? void 0 : resolve(h));
+  });
+  await page.goto(url);
+  const content = await waitForLoadPromise;
+  await browser.close();
+  return content;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getWebContent
+});

package/dist/cjs/utils/constants.d.ts

@@ -0,0 +1 @@
+export declare const iosUA = "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1";

package/dist/cjs/utils/constants.js

@@ -0,0 +1,29 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/constants.ts
+var constants_exports = {};
+__export(constants_exports, {
+  iosUA: () => iosUA
+});
+module.exports = __toCommonJS(constants_exports);
+var iosUA = "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1";
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  iosUA
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.29",
+  "version": "0.0.30",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",
@@ -39,6 +39,7 @@
     "dayjs": "^1.11.10",
     "html-entities": "^2.4.0",
     "lodash": "^4.17.21",
+    "playwright": "^1.41.2",
     "probe-image-size": "^7.2.3",
     "rss-parser": "^3.13.0",
     "yargs-parser": "^21.1.1",