ns-rss-spider 0.0.30 → 0.0.31

package/dist/cjs/index.d.ts

@@ -1,4 +1,3 @@
 export { parseRss } from "./parse";
 export { strategies } from "./strategy";
 export { parseContent } from "./parseContent";
-export { getWebContent } from './utils/browser';

package/dist/cjs/index.js

@@ -19,7 +19,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
-  getWebContent: () => import_browser.getWebContent,
   parseContent: () => import_parseContent.parseContent,
   parseRss: () => import_parse.parseRss,
   strategies: () => import_strategy.strategies
@@ -28,10 +27,8 @@ module.exports = __toCommonJS(src_exports);
 var import_parse = require("./parse");
 var import_strategy = require("./strategy");
 var import_parseContent = require("./parseContent");
-var import_browser = require("./utils/browser");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  getWebContent,
   parseContent,
   parseRss,
   strategies

package/dist/cjs/parsers/36kr.js

@@ -23,34 +23,42 @@ __export(kr_exports, {
 });
 module.exports = __toCommonJS(kr_exports);
 var import_zx = require("zx");
-var import_cheerio = require("cheerio");
 var import_constants = require("../utils/constants");
 var import_browser = require("../utils/browser");
 var _36kr = {
   parse: true,
   getThumbs: async (items) => {
+    var _a, _b, _c;
     const url = "https://m.36kr.com/";
-    const html = await (0, import_browser.getWebContent)(url, {
+    const initialState = await (0, import_browser.runInPage)(url, async (page) => {
+      const data = await page.evaluate(async () => window.initialState);
+      return data;
+    }, {
       userAgent: import_constants.iosUA
     }).catch((e) => {
       console.error(import_zx.chalk.red(`获取 ${url} 错误`), e);
     });
-    if (!html)
-      return void 0;
-    const $ = (0, import_cheerio.load)(html, null, false);
-    const kv = {};
-    $(".home-flow > .flow-list .article-item a").each((_, $a) => {
-      var _a, _b;
-      const thumb = $($a).find("img").attr("src");
-      const href = $($a).attr("href") || "";
-      const reg = /\/p\/([\d]+)/;
-      const articleId = (_a = reg.exec(href)) == null ? void 0 : _a[1];
-      const guid = (_b = items.find((a) => (a.guid || a.link || "").includes(`/p/${articleId}`))) == null ? void 0 : _b.guid;
-      if (guid && thumb) {
-        kv[guid] = thumb;
-      }
-    });
-    return kv;
+    console.log("initialState", initialState);
+    if ((_c = (_b = (_a = initialState == null ? void 0 : initialState.home) == null ? void 0 : _a.flow) == null ? void 0 : _b.itemList) == null ? void 0 : _c.length) {
+      const idThumbKv = initialState.home.flow.itemList.reduce((prev, item) => {
+        var _a2, _b2;
+        if ((item == null ? void 0 : item.itemId) && ((_a2 = item.templateMaterial) == null ? void 0 : _a2.widgetImage)) {
+          prev[item.itemId] = (_b2 = item.templateMaterial) == null ? void 0 : _b2.widgetImage;
+        }
+        return prev;
+      }, {});
+      const kv = {};
+      Object.keys(idThumbKv).forEach((articleId) => {
+        var _a2;
+        const thumb = idThumbKv[articleId];
+        const guid = (_a2 = items.find((a) => (a.guid || a.link || "").includes(`/p/${articleId}`))) == null ? void 0 : _a2.guid;
+        if (guid && thumb) {
+          kv[guid] = thumb;
+        }
+      });
+      return kv;
+    }
+    return void 0;
   }
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/utils/browser.d.ts

@@ -1,3 +1,29 @@
-export declare function getWebContent(url: string, options?: {
+import { Page } from "playwright";
+type RunOptions = {
     userAgent?: string;
-}): Promise<string>;
+};
+/**
+ * 在 url 中执行 fn，并获取 fn 的结果
+ * @param url
+ * @param fn
+ * @param options
+ * @returns
+ */
+export declare function runInPage<T>(url: string, fn: (page: Page) => Promise<T>, options?: RunOptions): Promise<T>;
+/**
+ * 根据 url 获取 content.
+ *
+ * 1. 支持获取前滚动页面
+ *
+ * @param url
+ * @param options
+ * @returns
+ */
+export declare function getPageHtml(url: string, options?: {
+    scroll?: {
+        offset?: number;
+        interval?: number;
+    };
+} & RunOptions): Promise<string>;
+export declare function scrollToBottom(page: Page, offset?: number, interval?: number): Promise<void>;
+export {};

package/dist/cjs/utils/browser.js

@@ -19,11 +19,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/utils/browser.ts
 var browser_exports = {};
 __export(browser_exports, {
-  getWebContent: () => getWebContent
+  getPageHtml: () => getPageHtml,
+  runInPage: () => runInPage,
+  scrollToBottom: () => scrollToBottom
 });
 module.exports = __toCommonJS(browser_exports);
 var import_playwright = require("playwright");
-async function getWebContent(url, options) {
+async function runInPage(url, fn, options) {
   const browser = await import_playwright.chromium.launch();
   const page = await browser.newPage({
     userAgent: options == null ? void 0 : options.userAgent
@@ -32,15 +34,47 @@ async function getWebContent(url, options) {
   const waitForLoadPromise = new Promise((r) => {
     resolve = r;
   });
-  page.on("load", (page2) => {
-    page2.locator("html").innerHTML().then((h) => resolve == null ? void 0 : resolve(h));
+  page.on("load", async (page2) => {
+    const r = await fn(page2);
+    resolve == null ? void 0 : resolve(r);
   });
   await page.goto(url);
   const content = await waitForLoadPromise;
   await browser.close();
   return content;
 }
+async function getPageHtml(url, options) {
+  return await runInPage(
+    url,
+    async (page) => {
+      var _a, _b;
+      if (options == null ? void 0 : options.scroll) {
+        await scrollToBottom(
+          page,
+          (_a = options == null ? void 0 : options.scroll) == null ? void 0 : _a.offset,
+          (_b = options == null ? void 0 : options.scroll) == null ? void 0 : _b.interval
+        );
+      }
+      const html = await page.locator("html").innerHTML();
+      return html;
+    },
+    options
+  );
+}
+async function scrollToBottom(page, offset = 300, interval = 10) {
+  const height = await page.evaluate(async () => {
+    const height2 = document.body.scrollHeight;
+    return height2;
+  });
+  console.log("height", height);
+  for (let i = 0; i < height; i += offset) {
+    page.mouse.wheel(0, i * offset);
+    await page.waitForTimeout(interval);
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  getWebContent
+  getPageHtml,
+  runInPage,
+  scrollToBottom
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.30",
+  "version": "0.0.31",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",