ns-rss-spider 0.0.33 → 1.0.0

package/dist/cjs/cli.js

@@ -40,7 +40,9 @@ async function main() {
   let rst;
   switch (cmd) {
     case "parse":
-      rst = await (0, import_index.parseRss)(rest.name, rest.feed, rest.server);
+      rst = await (0, import_index.parseRss)(rest.name, rest.feed, {
+        server: rest.server
+      });
       break;
     default:
       console.warn("未知命令", cmd);

package/dist/cjs/parse.d.ts

@@ -1,5 +1,8 @@
 import { ServerInfo } from "./upload";
-export declare function parseRss(name: string, feed: string, server?: ServerInfo): Promise<{
+export declare function parseRss(name: string, feed: string, options?: {
+    server?: ServerInfo;
+    take?: number;
+}): Promise<{
     contents: any[];
     thumbs?: Record<string, string>;
 }>;

package/dist/cjs/parse.js

@@ -32,20 +32,17 @@ __export(parse_exports, {
   parseRss: () => parseRss
 });
 module.exports = __toCommonJS(parse_exports);
-var import_rss_parser = __toESM(require("rss-parser"));
 var import_zx = require("zx");
 var import_strategy = require("./strategy");
 var import_parseContent = require("./parseContent");
 var import_upload = require("./upload");
 var import_axios = __toESM(require("axios"));
-async function parseRss(name, feed, server) {
-  const parser = new import_rss_parser.default({
-    customFields: {
-      item: ["source"]
-    }
-  });
+var import_parseRssList = require("./parseContent/parseRssList");
+async function parseRss(name, feed, options) {
+  const { server } = options || {};
+  console.log(import_zx.chalk.green("start", name, feed));
   console.log(import_zx.chalk.green("正在拉取 rss 列表"));
-  const result = await parser.parseURL(feed);
+  const result = await (0, import_parseRssList.parseRssList)(feed);
   if (!result.items.length) {
     throw Error("rss no conent");
   }
@@ -66,7 +63,8 @@ async function parseRss(name, feed, server) {
         item.guid = item.link;
       }
     });
-    for (let item of result.items) {
+    const items = (options == null ? void 0 : options.take) ? result.items.slice(0, options.take) : result.items;
+    for (let item of items) {
       console.log(import_zx.chalk.green(`正在解析文章 【${item.title}】`));
       if (!item.guid) {
         console.log(JSON.stringify(item));

package/dist/cjs/parseContent/parseRssList.d.ts

@@ -0,0 +1,6 @@
+import Parser from "rss-parser";
+export declare function parseRssList(feed: string): Promise<{
+    [key: string]: any;
+} & Parser.Output<{
+    source: any;
+}>>;

package/dist/cjs/parseContent/parseRssList.js

@@ -0,0 +1,50 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/parseRssList.ts
+var parseRssList_exports = {};
+__export(parseRssList_exports, {
+  parseRssList: () => parseRssList
+});
+module.exports = __toCommonJS(parseRssList_exports);
+var import_rss_parser = __toESM(require("rss-parser"));
+var import_request = require("../utils/request");
+async function parseRssList(feed) {
+  const xml = await (0, import_request.getPageHtmlByHttp)(feed);
+  const parser = new import_rss_parser.default({
+    customFields: {
+      item: ["source"]
+    }
+  });
+  const result = await parser.parseString(xml);
+  return result;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseRssList
+});

package/dist/cjs/parsers/36kr.js

@@ -38,7 +38,6 @@ var _36kr = {
     }).catch((e) => {
       console.error(import_zx.chalk.red(`获取 ${url} 错误`), e);
     });
-    console.log("initialState", initialState);
     if ((_c = (_b = (_a = initialState == null ? void 0 : initialState.home) == null ? void 0 : _a.flow) == null ? void 0 : _b.itemList) == null ? void 0 : _c.length) {
       const idThumbKv = initialState.home.flow.itemList.reduce((prev, item) => {
         var _a2, _b2;

package/dist/cjs/parsers/techrunch.d.ts

@@ -0,0 +1,2 @@
+import { ParseStrategy } from "../types";
+export declare const techrunch: ParseStrategy;

package/dist/cjs/parsers/techrunch.js

@@ -0,0 +1,63 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parsers/techrunch.ts
+var techrunch_exports = {};
+__export(techrunch_exports, {
+  techrunch: () => techrunch
+});
+module.exports = __toCommonJS(techrunch_exports);
+var import_zx = require("zx");
+var import_constants = require("../utils/constants");
+var import_browser = require("../utils/browser");
+var techrunch = {
+  parse: true,
+  fetcher: "http",
+  getContentElementFromArticle: ($) => {
+    const el = $(".article-content");
+    return el;
+  },
+  ignoreProbeImage: true,
+  getThumbs: async (items) => {
+    var _a, _b;
+    const url = "https://techcrunch.com/";
+    const initialState = await (0, import_browser.runInPage)(url, async (page) => {
+      const data = await page.evaluate(async () => window.tc_app_data);
+      return data;
+    }, {
+      userAgent: import_constants.iosUA
+    }).catch((e) => {
+      console.error(import_zx.chalk.red(`获取 ${url} 错误`), e);
+    });
+    if ((_b = (_a = initialState == null ? void 0 : initialState.entities) == null ? void 0 : _a.posts) == null ? void 0 : _b.length) {
+      const idThumbKv = initialState.entities.posts.reduce((prev, item) => {
+        var _a2;
+        if (((_a2 = item == null ? void 0 : item.guid) == null ? void 0 : _a2.rendered) && item.jetpack_featured_media_url) {
+          prev[item.guid.rendered] = item.jetpack_featured_media_url;
+        }
+        return prev;
+      }, {});
+      return idThumbKv;
+    }
+    return void 0;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  techrunch
+});

package/dist/cjs/strategy.js

@@ -27,12 +27,14 @@ var import_ifanr = require("./parsers/ifanr");
 var import_theverge = require("./parsers/theverge");
 var import_ithome = require("./parsers/ithome");
 var import_kr = require("./parsers/36kr");
+var import_techrunch = require("./parsers/techrunch");
 var strategies = {
   cnbeta: import_cnbeta.cnbeta,
   ifanr: import_ifanr.ifanr,
   theverge: import_theverge.theverge,
   ithome: import_ithome.ithome,
-  "36kr": import_kr._36kr
+  "36kr": import_kr._36kr,
+  techrunch: import_techrunch.techrunch
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/utils/browser.d.ts

@@ -19,11 +19,15 @@ export declare function runInPage<T>(url: string, fn: (page: Page) => Promise<T>
  * @param options
  * @returns
  */
-export declare function getPageHtml(url: string, options?: {
+export declare function getPageHtmlByBrowser(url: string, options?: {
     scroll?: {
         offset?: number;
         interval?: number;
     };
+    waitForElement?: {
+        selector: string;
+        timeout?: number;
+    };
 } & RunOptions): Promise<string>;
 export declare function scrollToBottom(page: Page, offset?: number, interval?: number): Promise<void>;
 export {};

package/dist/cjs/utils/browser.js

@@ -19,7 +19,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/utils/browser.ts
 var browser_exports = {};
 __export(browser_exports, {
-  getPageHtml: () => getPageHtml,
+  getPageHtmlByBrowser: () => getPageHtmlByBrowser,
   runInPage: () => runInPage,
   scrollToBottom: () => scrollToBottom
 });
@@ -43,11 +43,16 @@ async function runInPage(url, fn, options) {
   await browser.close();
   return content;
 }
-async function getPageHtml(url, options) {
+async function getPageHtmlByBrowser(url, options) {
   return await runInPage(
     url,
     async (page) => {
       var _a, _b;
+      if (options == null ? void 0 : options.waitForElement) {
+        await page.waitForSelector(options.waitForElement.selector, {
+          timeout: options.waitForElement.timeout || 1e3 * 3
+        });
+      }
       if (options == null ? void 0 : options.scroll) {
         await scrollToBottom(
           page,
@@ -74,7 +79,7 @@ async function scrollToBottom(page, offset = 300, interval = 10) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  getPageHtml,
+  getPageHtmlByBrowser,
   runInPage,
   scrollToBottom
 });

package/dist/cjs/utils/request.d.ts

@@ -0,0 +1 @@
+export declare function getPageHtmlByHttp(feed: string): Promise<string>;

package/dist/cjs/utils/request.js

@@ -0,0 +1,49 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/request.ts
+var request_exports = {};
+__export(request_exports, {
+  getPageHtmlByHttp: () => getPageHtmlByHttp
+});
+module.exports = __toCommonJS(request_exports);
+var import_axios = __toESM(require("axios"));
+var import_constants = require("./constants");
+async function getPageHtmlByHttp(feed) {
+  const res = await import_axios.default.get(feed, {
+    responseType: "text",
+    headers: {
+      "User-Agent": import_constants.iosUA
+    }
+  });
+  return res.data;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getPageHtmlByHttp
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.33",
+  "version": "1.0.0",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",