ns-rss-spider 0.0.2 → 0.0.3

package/dist/cjs/parse.d.ts

@@ -1 +1,10 @@
-export declare function parseRss(name: string, feed: string): Promise<(import("./types").SimpleContent | import("./types").RichContent)[]>;
+import { ServerInfo } from "./upload";
+export declare function parseRss(name: string, feed: string, server?: ServerInfo): Promise<import("./types").Article[] | {
+    storePath: any;
+    type: "simple" | "rich";
+    guid: string;
+    title: string;
+    link: string;
+    description: string;
+    pubDate: Date;
+}[]>;

package/dist/cjs/parse.js

@@ -36,22 +36,35 @@ var import_rss_parser = __toESM(require("rss-parser"));
 var import_zx = require("zx");
 var import_strategy = require("./strategy");
 var import_parseContent = require("./parseContent");
-async function parseRss(name, feed) {
+var import_upload = require("./upload");
+async function parseRss(name, feed, server) {
   const parser = new import_rss_parser.default();
-  console.log(import_zx.chalk.blue("正在拉取 rss 列表"));
+  console.log(import_zx.chalk.green("正在拉取 rss 列表"));
   const result = await parser.parseURL(feed);
   if (!result.items.length) {
     throw Error("rss no conent");
   }
   const contents = [];
   for (let item of result.items) {
-    console.log(import_zx.chalk.blue(`正在解析文章 【${item.title}】`));
+    console.log(import_zx.chalk.green(`正在解析文章 【${item.title}】`));
+    if (!item.guid) {
+      console.log(JSON.stringify(item));
+      throw Error(`item has no guid`);
+    }
     const content = await (0, import_parseContent.parseContent)(item, import_strategy.strategies[name]).catch((e) => console.error(import_zx.chalk.red("文章解析失败"), e.message));
     if (content) {
       contents.push(content);
     }
   }
-  return contents;
+  if (server) {
+    return await (0, import_upload.uploadContent)({
+      app: name,
+      items: contents,
+      ...server
+    });
+  } else {
+    return contents;
+  }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parseContent/index.d.ts

@@ -1,5 +1,5 @@
 import { Item as RssItem } from "rss-parser";
-import { Content, ParseStrategy } from "../types";
+import { Article, ParseStrategy } from "../types";
 /**
  * 解析文章内容
  * 1. 获取 html 片段。(rss、http、playwright)
@@ -13,4 +13,4 @@ import { Content, ParseStrategy } from "../types";
  * @param strategy
  * @returns
  */
-export declare function parseContent(item: RssItem, strategy?: ParseStrategy): Promise<Content>;
+export declare function parseContent(item: RssItem, strategy?: ParseStrategy): Promise<Article>;

package/dist/cjs/parseContent/index.js

@@ -45,26 +45,28 @@ async function parseContent(item, strategy) {
     $ = cheerio.load(item.content || "", {}, false);
     element = $.root();
   } else {
-    console.log(import_zx.chalk.blue("正在拉取文章内容"), item.link);
+    console.log(import_zx.chalk.green("正在拉取文章内容"), item.link);
     const article = await (0, import_getArticleHtml.getArticleHtml)(item.link, strategy.fetcher);
     $ = cheerio.load(article);
     element = ((_a = strategy.getContentHtmlFromArticle) == null ? void 0 : _a.call(strategy, $)) || $.root();
   }
-  console.log(import_zx.chalk.blue("正在预处理 html"));
+  console.log(import_zx.chalk.green("正在预处理 html"));
   (0, import_stripeHtml.stripeHtml)($, element);
   if (!(strategy == null ? void 0 : strategy.parse)) {
     return {
       type: "simple",
+      guid: item.guid,
       title: item.title,
+      link: item.link,
       description: ((_b = element.html()) == null ? void 0 : _b.trim()) || "",
       pubDate: new Date(item.pubDate)
     };
   }
-  console.log(import_zx.chalk.blue("正在解析文章内容"));
+  console.log(import_zx.chalk.green("正在解析文章内容"));
   const rich = await (0, import_parseContent.parseContent)($, element, item, strategy);
   let extra = {};
   if (strategy.getExtraItems) {
-    console.log(import_zx.chalk.blue("正在执行 getExtraItems"));
+    console.log(import_zx.chalk.green("正在执行 getExtraItems"));
     extra = await strategy.getExtraItems($, rich, item).catch((e) => {
       console.error(import_zx.chalk.red("getExtraItems 识别"), e);
     }) || {};

package/dist/cjs/parseContent/parseContent.d.ts

@@ -1,4 +1,4 @@
 import { Item as RssItem } from "rss-parser";
-import type { ParseStrategy, RichContent } from "../types";
+import type { ParseStrategy, RichArticle } from "../types";
 import { AnyNode, Cheerio, CheerioAPI } from "cheerio";
-export declare function parseContent($: CheerioAPI, $element: Cheerio<AnyNode>, item: RssItem, strategy?: ParseStrategy): Promise<RichContent>;
+export declare function parseContent($: CheerioAPI, $element: Cheerio<AnyNode>, item: RssItem, strategy?: ParseStrategy): Promise<RichArticle>;

package/dist/cjs/parseContent/parseContent.js

@@ -53,7 +53,7 @@ async function parseContent($, $element, item, strategy) {
     }
   });
   for (let item2 of srcs) {
-    console.log(import_zx.chalk.blue("正在解析图片", item2.src));
+    console.log(import_zx.chalk.green("正在解析图片", item2.src));
     const result = await (0, import_probe_image_size.default)(item2.src, {
       rejectUnauthorized: false,
       open_timeout: 1e3 * 5
@@ -95,7 +95,9 @@ async function parseContent($, $element, item, strategy) {
   });
   return {
     type: "rich",
+    guid: item.guid,
     title: item.title,
+    link: item.link,
     description: ((_a = item.contentSnippet) == null ? void 0 : _a.trim()) || "",
     content: (0, import_html_entities.decode)($element.html().trim()),
     pubDate: new Date(item.pubDate),

package/dist/cjs/types.d.ts

@@ -4,15 +4,17 @@ export interface ParseStrategy {
     parse: boolean;
     fetcher?: 'http' | 'playwright';
     getContentHtmlFromArticle?: (aritcle: CheerioAPI) => Cheerio<AnyNode>;
-    getExtraItems?: (html: CheerioAPI, current: RichContent, item: Item) => Promise<Record<string, any>>;
+    getExtraItems?: (html: CheerioAPI, current: RichArticle, item: Item) => Promise<Record<string, any>>;
 }
-export interface SimpleContent {
+export interface SimpleArticle {
     type: 'simple';
+    guid: string;
     title: string;
+    link: string;
     description: string;
     pubDate: Date;
 }
-export interface RichContentImage {
+export interface RichArticleImage {
     url: string;
     type?: string;
     width?: number;
@@ -22,9 +24,9 @@ export interface RichContentImage {
     heightUnit?: string;
     title?: string;
 }
-export interface RichContent extends Omit<SimpleContent, 'type'> {
+export interface RichArticle extends Omit<SimpleArticle, 'type'> {
     type: 'rich';
     content: string;
-    images: RichContentImage[];
+    images: RichArticleImage[];
 }
-export type Content = SimpleContent | RichContent;
+export type Article = SimpleArticle | RichArticle;

package/dist/cjs/upload/index.d.ts

@@ -0,0 +1,20 @@
+import { Article } from "../types";
+export interface ServerInfo {
+    getUploadApi?: string;
+    latest?: {
+        guid: string;
+        pubDate: Date;
+    };
+}
+export declare function uploadContent({ app, getUploadApi, latest, items, }: {
+    app: string;
+    items: Article[];
+} & ServerInfo): Promise<Article[] | {
+    storePath: any;
+    type: "simple" | "rich";
+    guid: string;
+    title: string;
+    link: string;
+    description: string;
+    pubDate: Date;
+}[]>;

package/dist/cjs/upload/index.js

@@ -0,0 +1,87 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/upload/index.ts
+var upload_exports = {};
+__export(upload_exports, {
+  uploadContent: () => uploadContent
+});
+module.exports = __toCommonJS(upload_exports);
+var import_lodash = require("lodash");
+var import_axios = __toESM(require("axios"));
+var import_dayjs = __toESM(require("dayjs"));
+var import_zx = require("zx");
+async function uploadContent({
+  app,
+  getUploadApi,
+  latest,
+  items
+}) {
+  var _a, _b;
+  const filterItems = latest ? items.filter((item) => {
+    return item.pubDate >= latest.pubDate && item.guid !== latest.guid;
+  }) : items;
+  if (!getUploadApi) {
+    return filterItems;
+  }
+  let result = [];
+  for (let item of filterItems) {
+    console.log(import_zx.chalk.green("正在获取上传地址", item.title));
+    const uploadRst = await import_axios.default.post(getUploadApi, {
+      app,
+      name: (0, import_dayjs.default)(item.pubDate).format("MM-DDTHH-mm-ss") + Math.random().toString(32).slice(2)
+    }, {
+      responseType: "json"
+    }).catch((e) => {
+      console.error(import_zx.chalk.red("get upload url error"), e);
+    });
+    if (!((_b = (_a = uploadRst == null ? void 0 : uploadRst.data) == null ? void 0 : _a.data) == null ? void 0 : _b.url)) {
+      console.error(import_zx.chalk.red(`upload error ${JSON.stringify(uploadRst)}`));
+      continue;
+    }
+    const url = uploadRst.data.data.url;
+    const storePath = uploadRst.data.data.storePath;
+    console.log(import_zx.chalk.green("正在上传", item.title, url));
+    const res = await import_axios.default.put(url, item).catch((e) => {
+      console.error(import_zx.chalk.red("upload oss error"), e);
+    });
+    if (!res || res.status !== 200) {
+      console.error(import_zx.chalk.red("upload status not 200"), res);
+      continue;
+    }
+    result.push({
+      ...(0, import_lodash.omit)(item, "content", "images"),
+      storePath
+    });
+  }
+  return result;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  uploadContent
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",
@@ -11,7 +11,7 @@
     "build:deps": "father prebundle",
     "check": "father doctor",
     "prepublishOnly": "father doctor && npm run build",
-    "pub": "npm run check && npm version patch && npm run build && npm publish --registry=https://registry.npmjs.org && git push origin master --tags && npm run sync"
+    "pub": "npm run check && npm version patch && npm run build && npm publish --registry=https://registry.npmjs.org && git push origin master --tags"
   },
   "bin": {
     "ns-rss-spider": "./dist/cjs/cli.js"
@@ -27,6 +27,7 @@
     "access": "public"
   },
   "devDependencies": {
+    "@types/lodash": "^4.14.202",
     "@types/probe-image-size": "^7.2.4",
     "father": "^4.4.0",
     "vitest": "^1.2.2"
@@ -34,7 +35,9 @@
   "dependencies": {
     "axios": "^1.6.7",
     "cheerio": "^1.0.0-rc.12",
+    "dayjs": "^1.11.10",
     "html-entities": "^2.4.0",
+    "lodash": "^4.17.21",
     "probe-image-size": "^7.2.3",
     "rss-parser": "^3.13.0",
     "yargs-parser": "^21.1.1",