ns-rss-spider 0.0.29 → 0.0.31

package/dist/cjs/parsers/36kr.js

@@ -22,8 +22,44 @@ __export(kr_exports, {
   _36kr: () => _36kr
 });
 module.exports = __toCommonJS(kr_exports);
+var import_zx = require("zx");
+var import_constants = require("../utils/constants");
+var import_browser = require("../utils/browser");
 var _36kr = {
-  parse: true
+  parse: true,
+  getThumbs: async (items) => {
+    var _a, _b, _c;
+    const url = "https://m.36kr.com/";
+    const initialState = await (0, import_browser.runInPage)(url, async (page) => {
+      const data = await page.evaluate(async () => window.initialState);
+      return data;
+    }, {
+      userAgent: import_constants.iosUA
+    }).catch((e) => {
+      console.error(import_zx.chalk.red(`获取 ${url} 错误`), e);
+    });
+    console.log("initialState", initialState);
+    if ((_c = (_b = (_a = initialState == null ? void 0 : initialState.home) == null ? void 0 : _a.flow) == null ? void 0 : _b.itemList) == null ? void 0 : _c.length) {
+      const idThumbKv = initialState.home.flow.itemList.reduce((prev, item) => {
+        var _a2, _b2;
+        if ((item == null ? void 0 : item.itemId) && ((_a2 = item.templateMaterial) == null ? void 0 : _a2.widgetImage)) {
+          prev[item.itemId] = (_b2 = item.templateMaterial) == null ? void 0 : _b2.widgetImage;
+        }
+        return prev;
+      }, {});
+      const kv = {};
+      Object.keys(idThumbKv).forEach((articleId) => {
+        var _a2;
+        const thumb = idThumbKv[articleId];
+        const guid = (_a2 = items.find((a) => (a.guid || a.link || "").includes(`/p/${articleId}`))) == null ? void 0 : _a2.guid;
+        if (guid && thumb) {
+          kv[guid] = thumb;
+        }
+      });
+      return kv;
+    }
+    return void 0;
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parsers/cnbeta.js

@@ -35,6 +35,7 @@ module.exports = __toCommonJS(cnbeta_exports);
 var import_axios = __toESM(require("axios"));
 var import_zx = require("zx");
 var import_cheerio = require("cheerio");
+var import_constants = require("../utils/constants");
 var cnbeta = {
   parse: true,
   fetcher: "http",
@@ -59,7 +60,10 @@ var cnbeta = {
   ignoreProbeImage: true,
   getThumbs: async (items) => {
     const res = await import_axios.default.get("https://m.cnbeta.com.tw/", {
-      responseType: "text"
+      responseType: "text",
+      headers: {
+        "User-Agent": import_constants.iosUA
+      }
     }).catch((e) => {
       console.error(import_zx.chalk.red("获取 m.cnbeta.com.tw 错误"), e);
     });

package/dist/cjs/parsers/ithome.js

@@ -36,6 +36,7 @@ var import_utils = require("../parseContent/utils");
 var import_axios = __toESM(require("axios"));
 var import_zx = require("zx");
 var import_cheerio = require("cheerio");
+var import_constants = require("../utils/constants");
 var ithome = {
   parse: true,
   fetcher: "http",
@@ -64,7 +65,10 @@ var ithome = {
   },
   getThumbs: async (articles) => {
     const res = await import_axios.default.get("https://m.ithome.com/", {
-      responseType: "text"
+      responseType: "text",
+      headers: {
+        "User-Agent": import_constants.iosUA
+      }
     }).catch((e) => {
       console.error(import_zx.chalk.red("获取 https://m.ithome.com/ 错误"), e);
     });

package/dist/cjs/utils/browser.d.ts

@@ -0,0 +1,29 @@
+import { Page } from "playwright";
+type RunOptions = {
+    userAgent?: string;
+};
+/**
+ * 在 url 中执行 fn，并获取 fn 的结果
+ * @param url
+ * @param fn
+ * @param options
+ * @returns
+ */
+export declare function runInPage<T>(url: string, fn: (page: Page) => Promise<T>, options?: RunOptions): Promise<T>;
+/**
+ * 根据 url 获取 content.
+ *
+ * 1. 支持获取前滚动页面
+ *
+ * @param url
+ * @param options
+ * @returns
+ */
+export declare function getPageHtml(url: string, options?: {
+    scroll?: {
+        offset?: number;
+        interval?: number;
+    };
+} & RunOptions): Promise<string>;
+export declare function scrollToBottom(page: Page, offset?: number, interval?: number): Promise<void>;
+export {};

package/dist/cjs/utils/browser.js

@@ -0,0 +1,80 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/browser.ts
+var browser_exports = {};
+__export(browser_exports, {
+  getPageHtml: () => getPageHtml,
+  runInPage: () => runInPage,
+  scrollToBottom: () => scrollToBottom
+});
+module.exports = __toCommonJS(browser_exports);
+var import_playwright = require("playwright");
+async function runInPage(url, fn, options) {
+  const browser = await import_playwright.chromium.launch();
+  const page = await browser.newPage({
+    userAgent: options == null ? void 0 : options.userAgent
+  });
+  let resolve;
+  const waitForLoadPromise = new Promise((r) => {
+    resolve = r;
+  });
+  page.on("load", async (page2) => {
+    const r = await fn(page2);
+    resolve == null ? void 0 : resolve(r);
+  });
+  await page.goto(url);
+  const content = await waitForLoadPromise;
+  await browser.close();
+  return content;
+}
+async function getPageHtml(url, options) {
+  return await runInPage(
+    url,
+    async (page) => {
+      var _a, _b;
+      if (options == null ? void 0 : options.scroll) {
+        await scrollToBottom(
+          page,
+          (_a = options == null ? void 0 : options.scroll) == null ? void 0 : _a.offset,
+          (_b = options == null ? void 0 : options.scroll) == null ? void 0 : _b.interval
+        );
+      }
+      const html = await page.locator("html").innerHTML();
+      return html;
+    },
+    options
+  );
+}
+async function scrollToBottom(page, offset = 300, interval = 10) {
+  const height = await page.evaluate(async () => {
+    const height2 = document.body.scrollHeight;
+    return height2;
+  });
+  console.log("height", height);
+  for (let i = 0; i < height; i += offset) {
+    page.mouse.wheel(0, i * offset);
+    await page.waitForTimeout(interval);
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getPageHtml,
+  runInPage,
+  scrollToBottom
+});

package/dist/cjs/utils/constants.d.ts

@@ -0,0 +1 @@
+export declare const iosUA = "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1";

package/dist/cjs/utils/constants.js

@@ -0,0 +1,29 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/constants.ts
+var constants_exports = {};
+__export(constants_exports, {
+  iosUA: () => iosUA
+});
+module.exports = __toCommonJS(constants_exports);
+var iosUA = "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1";
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  iosUA
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.29",
+  "version": "0.0.31",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",
@@ -39,6 +39,7 @@
     "dayjs": "^1.11.10",
     "html-entities": "^2.4.0",
     "lodash": "^4.17.21",
+    "playwright": "^1.41.2",
     "probe-image-size": "^7.2.3",
     "rss-parser": "^3.13.0",
     "yargs-parser": "^21.1.1",