ns-rss-spider 1.1.5 → 1.1.8

package/dist/cjs/parseContent/index.js

@@ -49,19 +49,50 @@ async function parseContent(item, strategy) {
     console.log(import_zx.chalk.green("正在拉取文章内容"), item.link);
     const article = await (0, import_getArticleHtml.getArticleHtml)(item.link, strategy.fetcher);
     $ = cheerio.load(article);
-    element = ((_a = strategy.getContentElementFromArticle) == null ? void 0 : _a.call(strategy, $)) || $.root();
+    element = ((_a = strategy.getContentElementFromArticle) == null ? void 0 : _a.call(strategy, $)) || $.root().find("body");
+  }
+  let newsletter;
+  if ((strategy == null ? void 0 : strategy.articleType) === "newsletter") {
+    console.log("解析 newsletter");
+    const ns = await strategy.run($, element).catch((e) => {
+      console.error(import_zx.chalk.red("parse topic error"));
+      console.error(e);
+      return void 0;
+    });
+    if (!ns || !ns.topics.length) {
+      console.error(import_zx.chalk.red("未解析到 topics"));
+      console.log(ns);
+    } else {
+      newsletter = ns;
+    }
   }
   console.log(import_zx.chalk.green("正在预处理 html"));
-  (0, import_stripeHtml.stripeHtml)($, element);
-  if (!(strategy == null ? void 0 : strategy.parse)) {
+  (0, import_stripeHtml.stripeHtml)($, element, strategy);
+  const base = {
+    ...(0, import_getBasicFromItem.getBasicFromItem)(item),
+    content: ((_b = element.html()) == null ? void 0 : _b.trim()) || ""
+  };
+  if (!(strategy == null ? void 0 : strategy.parse) || strategy.articleType === "newsletter" && !newsletter) {
     return {
       type: "simple",
-      ...(0, import_getBasicFromItem.getBasicFromItem)(item),
-      content: ((_b = element.html()) == null ? void 0 : _b.trim()) || ""
+      ...base
+    };
+  }
+  if (strategy.articleType === "newsletter") {
+    return {
+      type: "newsletter",
+      ...base,
+      newsletter
     };
   }
   console.log(import_zx.chalk.green("正在解析文章内容"));
   const rich = await (0, import_parseContent.parseContent)($, element, item, strategy);
+  if (!rich) {
+    return {
+      type: "simple",
+      ...base
+    };
+  }
   let extra = {};
   if (rich.type === "rich") {
     if (strategy.getExtraItems) {

package/dist/cjs/parseContent/newsletter/parseHtml.d.ts

@@ -0,0 +1,4 @@
+import { AnyNode, Cheerio, CheerioAPI } from "cheerio";
+export declare const parseHtml: ($: CheerioAPI, $el: Cheerio<AnyNode>, options?: {
+    removeLink?: boolean | undefined;
+}) => string;

package/dist/cjs/parseContent/newsletter/parseHtml.js

@@ -0,0 +1,54 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/parseHtml.ts
+var parseHtml_exports = {};
+__export(parseHtml_exports, {
+  parseHtml: () => parseHtml
+});
+module.exports = __toCommonJS(parseHtml_exports);
+var import_html_entities = require("html-entities");
+var parseHtml = ($, $el, options = {}) => {
+  const { removeLink } = options;
+  $el.find("*").each(function() {
+    const tag = this.tagName.toLowerCase();
+    if (tag === "a") {
+      if (removeLink) {
+        $(this).replaceWith($(`<span class="ns-desc-link" data-href="${this.attribs.href}">${$(this).text()}</span>`));
+      } else {
+        this.attribs = {
+          href: this.attribs.href
+        };
+      }
+    } else if (tag === "image") {
+      this.attribs = {
+        src: this.attribs.src
+      };
+    } else {
+      this.attribs = {};
+    }
+    if (["div", "span", "p", "ol", "ul", "blockquote", "a", "hr"].includes(tag) && $(this).children().length === 0 && $(this).text().trim() === "") {
+      $(this).remove();
+    }
+  });
+  return (0, import_html_entities.decode)($el.html().trim());
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseHtml
+});

package/dist/cjs/parseContent/newsletter/parseNewsLetter.d.ts

@@ -0,0 +1,4 @@
+import { AnyNode, Cheerio, CheerioAPI } from "cheerio";
+import { ParseTpl } from "./types";
+import { SnapshotResult } from "./types";
+export declare function parseNewsLetter(tpl: ParseTpl, $: CheerioAPI, $root: Cheerio<AnyNode>): SnapshotResult;

package/dist/cjs/parseContent/newsletter/parseNewsLetter.js

@@ -0,0 +1,50 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/parseNewsLetter.ts
+var parseNewsLetter_exports = {};
+__export(parseNewsLetter_exports, {
+  parseNewsLetter: () => parseNewsLetter
+});
+module.exports = __toCommonJS(parseNewsLetter_exports);
+var import_parseHtml = require("./parseHtml");
+function parseNewsLetter(tpl, $, $root) {
+  let rst = {
+    name: "",
+    topics: []
+  };
+  if (tpl.description) {
+    const $desc = $root.find(tpl.description);
+    if ($desc.html()) {
+      const desc = (0, import_parseHtml.parseHtml)($, $root.find(tpl.description), {
+        removeLink: tpl.descriptionRemoveLink
+      });
+      if (desc) {
+        rst.description = desc;
+      }
+    }
+  }
+  if (tpl.image) {
+    rst.image = $root.find(tpl.image).first().attr("src");
+  }
+  return rst;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseNewsLetter
+});

package/dist/cjs/parseContent/newsletter/parseNoteItem.d.ts

@@ -0,0 +1,4 @@
+import type { AnyNode, Cheerio, CheerioAPI } from "cheerio";
+import { ParseTpl, ParseTplTopicCommon } from "./types";
+import { SnapshotNote } from './types';
+export declare function parseNoteItem($: CheerioAPI, $el: Cheerio<AnyNode>, tpl: ParseTplTopicCommon, transform?: ParseTpl["transform"]): SnapshotNote | undefined;

package/dist/cjs/parseContent/newsletter/parseNoteItem.js

@@ -0,0 +1,59 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/parseNoteItem.ts
+var parseNoteItem_exports = {};
+__export(parseNoteItem_exports, {
+  parseNoteItem: () => parseNoteItem
+});
+module.exports = __toCommonJS(parseNoteItem_exports);
+var import_parseHtml = require("./parseHtml");
+function parseNoteItem($, $el, tpl, transform) {
+  const $link = $el.find(tpl.noteLink).first();
+  const url = $link.attr("href");
+  if (!url) {
+    return void 0;
+  }
+  const $tag = $el.find(tpl.noteTag).first();
+  const tag = $tag.text().trim();
+  const $desc = $el.find(tpl.noteBody).first();
+  let title = tpl.noteNoTitle ? "" : $link.text().trim();
+  title = (transform == null ? void 0 : transform("noteTitle", title)) || title;
+  let descriptionHtml;
+  if (tpl.noteRichDesc) {
+    descriptionHtml = (0, import_parseHtml.parseHtml)($, $desc, {
+      removeLink: tpl.noteRichDescRemoveLink
+    });
+  }
+  let description = $desc.text().trim() || "";
+  description = (transform == null ? void 0 : transform("noteDesc", description)) || description;
+  const rst = {
+    title: tpl.noteNoTitle ? "" : $link.text().trim(),
+    url,
+    description,
+    tag
+  };
+  if (descriptionHtml) {
+    rst.descriptionHtml = descriptionHtml;
+  }
+  return rst;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseNoteItem
+});

package/dist/cjs/parseContent/newsletter/plainToGroup.d.ts

@@ -0,0 +1,8 @@
+import { AnyNode, Cheerio, CheerioAPI } from "cheerio";
+export declare const emptyGroup: string;
+export declare const emptyNote: string;
+export declare function plainToGroup({ root, isTopicTitle, isTitle, }: {
+    root: Cheerio<AnyNode>;
+    isTopicTitle: (node: Cheerio<AnyNode>) => boolean;
+    isTitle: (node: Cheerio<AnyNode>) => boolean;
+}): CheerioAPI;

package/dist/cjs/parseContent/newsletter/plainToGroup.js

@@ -0,0 +1,73 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/plainToGroup.ts
+var plainToGroup_exports = {};
+__export(plainToGroup_exports, {
+  emptyGroup: () => emptyGroup,
+  emptyNote: () => emptyNote,
+  plainToGroup: () => plainToGroup
+});
+module.exports = __toCommonJS(plainToGroup_exports);
+var import_cheerio = require("cheerio");
+var emptyGroup = `
+<div class='ns-topic'>
+  <div class='ns-topic-title'></div>
+  <div class='ns-topic-content'></div>
+</div>
+`.trim();
+var emptyNote = `
+<div class='ns-note'>
+  <div class='ns-note-title'></div>
+  <div class='ns-note-content'></div>
+</div>
+`.trim();
+function plainToGroup({
+  root,
+  isTopicTitle,
+  isTitle
+}) {
+  const $ = (0, import_cheerio.load)("<body></body>");
+  let currentTopic = $(emptyGroup);
+  let currentNote = null;
+  root.children().each((_i, node) => {
+    const $node = $(node);
+    if (isTopicTitle($node)) {
+      currentTopic = $(emptyGroup);
+      currentTopic.find(".ns-topic-title").append($node.clone());
+      $("body").append(currentTopic);
+    } else if (isTitle($node)) {
+      currentNote = $(emptyNote);
+      currentNote.find(".ns-note-title").append($node.clone());
+      currentTopic.find(".ns-topic-content").append(currentNote);
+    } else {
+      if (currentNote) {
+        currentNote.find(".ns-note-content").append($node.clone());
+      } else {
+        console.error("invalid currentNote");
+      }
+    }
+  });
+  return $;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  emptyGroup,
+  emptyNote,
+  plainToGroup
+});

package/dist/cjs/parseContent/newsletter/tpl.d.ts

@@ -0,0 +1,11 @@
+import type { AnyNode, Cheerio, CheerioAPI } from "cheerio";
+import type { SnapshotResult } from "./types";
+import { ParseTpl } from "./types";
+/**
+ * linear 和 group 并存场景下的解析思路
+ *
+ * 1. 通过 selector 选出 linerTitle、linearItem、group、linerTitle、linearItem、group... 序列
+ * 2. 遇到一个 linearTitle 或者一个 group，就组成一个新的 group
+ *
+ */
+export declare const parse: (tpl: ParseTpl, $: CheerioAPI, el: Cheerio<AnyNode>) => SnapshotResult;

package/dist/cjs/parseContent/newsletter/tpl.js

@@ -0,0 +1,86 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/tpl.ts
+var tpl_exports = {};
+__export(tpl_exports, {
+  parse: () => parse
+});
+module.exports = __toCommonJS(tpl_exports);
+var import_parseNewsLetter = require("./parseNewsLetter");
+var import_parseNoteItem = require("./parseNoteItem");
+var parse = (tpl, $, el) => {
+  let rst = (0, import_parseNewsLetter.parseNewsLetter)(tpl, $, $(el));
+  let selectors = [];
+  if (tpl.linearTopic) {
+    selectors = selectors.concat([tpl.linearTopic.title, tpl.linearTopic.item]);
+  }
+  if (tpl.groupTopic) {
+    selectors.push(`${tpl.groupTopic.group}`.trim());
+  }
+  const topicFlags = $(el).find(selectors.join(","));
+  let t = [];
+  topicFlags.each((_, el2) => {
+    var _a, _b, _c, _d;
+    if (el2.nodeType !== 1) {
+      throw Error("invalid title or item: nodeType !== 1");
+    }
+    const $el = $(el2);
+    const isLinearTitle = (_a = tpl.linearTopic) == null ? void 0 : _a.matchTitle($el);
+    const isLinearItem = (_b = tpl.linearTopic) == null ? void 0 : _b.matchItem($el);
+    if (isLinearTitle) {
+      let title = $el.text().trim();
+      title = ((_c = tpl.transform) == null ? void 0 : _c.call(tpl, "noteTitle", title)) || title;
+      t.push({
+        title,
+        notes: []
+      });
+    } else if (isLinearItem) {
+      if (!t.length) {
+        t.push({
+          title: "",
+          notes: []
+        });
+      }
+      const item = (0, import_parseNoteItem.parseNoteItem)($, $el, tpl.linearTopic, tpl.transform);
+      if (item) {
+        t[t.length - 1].notes.push(item);
+      }
+    } else {
+      const $group = $el;
+      const gTpl = tpl.groupTopic;
+      let title = $group.find(gTpl.title).first().text().trim();
+      title = ((_d = tpl.transform) == null ? void 0 : _d.call(tpl, "noteTitle", title)) || title;
+      const notes = Array.from($group.find(gTpl.item)).map((item) => {
+        const $item = $(item);
+        return (0, import_parseNoteItem.parseNoteItem)($, $item, gTpl, tpl.transform);
+      }).filter((n) => !!n);
+      t.push({
+        title,
+        notes
+      });
+    }
+  });
+  t = t.filter((t2) => t2.notes.length > 0);
+  rst.topics = t;
+  return rst;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parse
+});

package/dist/cjs/parseContent/newsletter/types.d.ts

@@ -0,0 +1,46 @@
+import { AnyNode, Cheerio } from "cheerio";
+export type Selector = string;
+export interface ParseTplTopicCommon {
+    root?: Selector;
+    title: Selector;
+    item: Selector;
+    noteLink: Selector;
+    noteBody?: Selector;
+    noteFrom?: Selector;
+    noteNoTitle?: boolean;
+    noteTag?: Selector;
+    noteRichDesc?: boolean;
+    noteRichDescRemoveLink?: boolean;
+}
+export interface ParseTpl {
+    description?: Selector;
+    descriptionRemoveLink?: boolean;
+    image?: Selector;
+    transform?: (type: "date" | "noteTitle" | "noteDesc", content: string) => string | undefined;
+    linearTopic?: ParseTplTopicCommon & {
+        matchTitle: (el: Cheerio<AnyNode>) => boolean;
+        matchItem: (el: Cheerio<AnyNode>) => boolean;
+    };
+    groupTopic?: {
+        group: Selector;
+    } & ParseTplTopicCommon;
+}
+export interface SnapshotResult {
+    name: string;
+    description?: string;
+    image?: string;
+    topics: SnapshotTopic[];
+}
+export interface SnapshotTopic {
+    title: string;
+    description?: string;
+    image?: string;
+    notes: SnapshotNote[];
+}
+export interface SnapshotNote {
+    url: string;
+    title: string;
+    description: string;
+    descriptionHtml?: string;
+    tag?: string;
+}

package/dist/cjs/parseContent/newsletter/types.js

@@ -0,0 +1,17 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parseContent/newsletter/types.ts
+var types_exports = {};
+module.exports = __toCommonJS(types_exports);

package/dist/cjs/parseContent/parseContent.d.ts

@@ -1,3 +1,3 @@
-import type { Article, ParseStrategy, RssItem } from "../types";
+import type { Article, ParseStrategyArticle, RssItem } from "../types";
 import { AnyNode, Cheerio, CheerioAPI } from "cheerio";
-export declare function parseContent($: CheerioAPI, $element: Cheerio<AnyNode>, item: RssItem, strategy?: ParseStrategy): Promise<Article>;
+export declare function parseContent($: CheerioAPI, $element: Cheerio<AnyNode>, item: RssItem, strategy?: ParseStrategyArticle): Promise<Article | undefined>;

package/dist/cjs/parseContent/parseContent.js

@@ -47,7 +47,7 @@ async function parseContent($, $element, item, strategy) {
     if (node.is("img")) {
       const src = node.attr("src");
       const title = node.attr("title") || void 0;
-      if (src) {
+      if (src && !src.startsWith("data:")) {
         srcs.push({
           src,
           title
@@ -55,6 +55,7 @@ async function parseContent($, $element, item, strategy) {
       }
     }
   });
+  console.log("开始解析图片，数目:", srcs.length);
   for (let item2 of srcs) {
     if (strategy == null ? void 0 : strategy.ignoreProbeImage) {
       images.push({
@@ -109,11 +110,7 @@ async function parseContent($, $element, item, strategy) {
   if (!content) {
     console.error(import_zx.chalk.red("解析 html 内容出错, html 如下"));
     console.log($.html());
-    return {
-      type: "simple",
-      content: "",
-      ...(0, import_getBasicFromItem.getBasicFromItem)(item)
-    };
+    return void 0;
   }
   return {
     type: "rich",

package/dist/cjs/parseContent/stripeHtml.d.ts

@@ -1,2 +1,3 @@
 import type { Cheerio, AnyNode, CheerioAPI } from "cheerio";
-export declare function stripeHtml($: CheerioAPI, element: Cheerio<AnyNode>): void;
+import type { ParseStrategy } from "../types";
+export declare function stripeHtml($: CheerioAPI, element: Cheerio<AnyNode>, strategy?: ParseStrategy): void;

package/dist/cjs/parseContent/stripeHtml.js

@@ -23,7 +23,7 @@ __export(stripeHtml_exports, {
 });
 module.exports = __toCommonJS(stripeHtml_exports);
 var import_utils = require("./utils");
-function stripeHtml($, element) {
+function stripeHtml($, element, strategy) {
   $(element).find("script").remove();
   (0, import_utils.walk_the_DOM)($, element, (el) => {
     for (let attr of [
@@ -39,6 +39,11 @@ function stripeHtml($, element) {
       $(el).removeAttr("target");
     }
     if ($(el).is("img")) {
+      if (strategy == null ? void 0 : strategy.imageSrcAttr) {
+        const src2 = $(el).attr(strategy.imageSrcAttr);
+        $(el).attr("src", src2);
+        ;
+      }
       const src = $(el).attr("src");
       if (src == null ? void 0 : src.startsWith("//")) {
         console.log("img `//` 开头域名添加 https", src);

package/dist/cjs/parsers/avanderlee.js

@@ -23,7 +23,16 @@ __export(avanderlee_exports, {
 });
 module.exports = __toCommonJS(avanderlee_exports);
 var avanderlee = {
-  parse: false
+  parse: true,
+  fetcher: "http",
+  imageSrcAttr: "data-lazy-src",
+  getContentElementFromArticle: ($) => {
+    const el = $("#content > article > section.post-content");
+    for (let sel of [".gotngnp", ".article-inline-newsletter-container", "noscript", 'a > svg[aria-hidden="true"]']) {
+      $(el).find(sel).remove();
+    }
+    return el;
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/parsers/cnbeta.js

@@ -57,7 +57,7 @@ var cnbeta = {
  阅读全文`, "")
     };
   },
-  //ignoreProbeImage: true,
+  ignoreProbeImage: true,
   getThumbs: async (items) => {
     const res = await import_axios.default.get("http://m.cnbeta.com.tw/", {
       responseType: "text",

package/dist/cjs/parsers/cssweekly.d.ts

@@ -0,0 +1,4 @@
+import { ParseTpl } from "../parseContent/newsletter/types";
+import { ParseStrategy } from "../types";
+export declare const tpl: ParseTpl;
+export declare const cssweekly: ParseStrategy;

package/dist/cjs/parsers/cssweekly.js

@@ -0,0 +1,51 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parsers/cssweekly.ts
+var cssweekly_exports = {};
+__export(cssweekly_exports, {
+  cssweekly: () => cssweekly,
+  tpl: () => tpl
+});
+module.exports = __toCommonJS(cssweekly_exports);
+var import_tpl = require("../parseContent/newsletter/tpl");
+var tpl = {
+  description: ".newsletter-intro",
+  groupTopic: {
+    group: "section.newsletter-section",
+    title: "h2.section-title",
+    item: "article.newsletter-article",
+    noteLink: ".newsletter-header h2.article-title a",
+    noteBody: "p",
+    noteRichDesc: true
+  }
+};
+var cssweekly = {
+  parse: true,
+  fetcher: "http",
+  articleType: "newsletter",
+  run: async ($, el) => {
+    const rst = await (0, import_tpl.parse)(tpl, $, el);
+    return rst;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  cssweekly,
+  tpl
+});

package/dist/cjs/parsers/iosdevweekly.d.ts

@@ -0,0 +1,4 @@
+import { ParseTpl } from "../parseContent/newsletter/types";
+import { ParseStrategy } from "../types";
+export declare const tpl: ParseTpl;
+export declare const iosdevweekly: ParseStrategy;

package/dist/cjs/parsers/iosdevweekly.js

@@ -0,0 +1,56 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parsers/iosdevweekly.ts
+var iosdevweekly_exports = {};
+__export(iosdevweekly_exports, {
+  iosdevweekly: () => iosdevweekly,
+  tpl: () => tpl
+});
+module.exports = __toCommonJS(iosdevweekly_exports);
+var import_tpl = require("../parseContent/newsletter/tpl");
+var tpl = {
+  description: ".category.cc-comment #comment > .item--issue",
+  descriptionRemoveLink: true,
+  groupTopic: {
+    group: ".issue__body > .category",
+    title: "h2.category__title",
+    item: ".item.item--issue",
+    noteLink: "h3.item__title > a",
+    noteBody: "p",
+    noteRichDesc: true,
+    noteRichDescRemoveLink: true
+  }
+};
+var iosdevweekly = {
+  parse: true,
+  fetcher: "http",
+  articleType: "newsletter",
+  getContentElementFromArticle($) {
+    return $("article.issue");
+  },
+  run: async ($, el) => {
+    const rst = await (0, import_tpl.parse)(tpl, $, el);
+    return rst;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  iosdevweekly,
+  tpl
+});

package/dist/cjs/parsers/javascriptweekly.d.ts

@@ -0,0 +1,8 @@
+import { ParseTpl } from "../parseContent/newsletter/types";
+import { ParseStrategy } from "../types";
+import { CheerioAPI } from "cheerio";
+export declare const tpl: ParseTpl;
+export declare function transform($: CheerioAPI): CheerioAPI;
+export declare function transformFromInDesc($: CheerioAPI): CheerioAPI;
+export declare function transformExceptionGroup($: CheerioAPI): CheerioAPI;
+export declare const javascriptweekly: ParseStrategy;

package/dist/cjs/parsers/javascriptweekly.js

@@ -0,0 +1,124 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/parsers/javascriptweekly.ts
+var javascriptweekly_exports = {};
+__export(javascriptweekly_exports, {
+  javascriptweekly: () => javascriptweekly,
+  tpl: () => tpl,
+  transform: () => transform,
+  transformExceptionGroup: () => transformExceptionGroup,
+  transformFromInDesc: () => transformFromInDesc
+});
+module.exports = __toCommonJS(javascriptweekly_exports);
+var import_tpl = require("../parseContent/newsletter/tpl");
+var import_arraySplit = require("../utils/arraySplit");
+var r = "#content";
+var tpl = {
+  image: `${r} .el-fullwidthimage img`,
+  transform: (type, desc) => {
+    var _a;
+    if (type === "noteDesc") {
+      let d = desc.trim();
+      if (d.startsWith("—")) {
+        d = d.substring(1).trim();
+      }
+      return d;
+    } else if (type === "noteTitle" && desc.endsWith(":")) {
+      return desc.substring(0, desc.length - 1);
+    } else if (type === "date") {
+      return ((_a = desc.split("—")[1]) == null ? void 0 : _a.trim()) || "";
+    }
+  },
+  linearTopic: {
+    title: `${r} > .el-heading`,
+    matchTitle: (el) => el.hasClass("el-heading"),
+    item: `${r} > .el-item, ${r} > .miniitem.item`,
+    matchItem: (el) => ["el-item", "miniitem"].some((cls) => el.hasClass(cls)),
+    noteLink: ".desc a",
+    noteBody: ".desc",
+    noteFrom: ".name",
+    noteTag: ".tag-sponsor",
+    noteRichDesc: true,
+    noteRichDescRemoveLink: true
+  },
+  groupTopic: {
+    group: `${r} .content.el-md, ${r} > .el-subtable table.content.el-content.briefs`,
+    title: "tbody > tr > td > p",
+    item: "tbody > tr > td > ul > li",
+    noteLink: "a",
+    noteBody: "*",
+    noteTag: ".tag-sponsor",
+    noteRichDesc: true,
+    noteRichDescRemoveLink: true
+    //noteDescIncludeTitle: true,
+  }
+};
+function transform($) {
+  transformExceptionGroup($);
+  transformFromInDesc($);
+  return $;
+}
+function transformFromInDesc($) {
+  const items = $(tpl.linearTopic.item);
+  items.each((i, item) => {
+    const $item = $(item);
+    if ($item.hasClass("miniitem")) {
+      $item.find("tbody > tr > td > p.desc > .name").last().remove();
+    }
+  });
+  return $;
+}
+function transformExceptionGroup($) {
+  const groups = $(tpl.groupTopic.group);
+  groups.each((i, group) => {
+    const $group = $(group).find("tbody > tr > td");
+    if (Array.from($group.children()).map((el) => el.tagName.toLowerCase()).join("") === "pp") {
+      const noteEl = $group.children().last();
+      const lis = (0, import_arraySplit.arraySplit)(Array.from(noteEl.contents()), (el) => el.nodeType === 1 && el.tagName === "br");
+      noteEl.replaceWith($("<ul></ul>")).html("");
+      lis.forEach((li) => {
+        const l = $("<li></li>");
+        const p = $("<p></p>");
+        li.forEach((el) => {
+          p.append(el);
+        });
+        l.append(p);
+        $group.children().last().append(l);
+      });
+    }
+  });
+  return $;
+}
+var javascriptweekly = {
+  parse: true,
+  fetcher: "http",
+  articleType: "newsletter",
+  run: async ($, el) => {
+    const rst = await (0, import_tpl.parse)(tpl, transform($), el);
+    return rst;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  javascriptweekly,
+  tpl,
+  transform,
+  transformExceptionGroup,
+  transformFromInDesc
+});

package/dist/cjs/strategy.js

@@ -33,6 +33,9 @@ var import_tmtpost = require("./parsers/tmtpost");
 var import_leiphone = require("./parsers/leiphone");
 var import_oschina = require("./parsers/oschina");
 var import_avanderlee = require("./parsers/avanderlee");
+var import_iosdevweekly = require("./parsers/iosdevweekly");
+var import_cssweekly = require("./parsers/cssweekly");
+var import_javascriptweekly = require("./parsers/javascriptweekly");
 var strategies = {
   cnbeta: import_cnbeta.cnbeta,
   ifanr: import_ifanr.ifanr,
@@ -44,7 +47,13 @@ var strategies = {
   tmtpost: import_tmtpost.tmtpost,
   leiphone: import_leiphone.leiphone,
   oschina: import_oschina.oschina,
-  avanderlee: import_avanderlee.avanderlee
+  avanderlee: import_avanderlee.avanderlee,
+  iosdevweekly: import_iosdevweekly.iosdevweekly,
+  cssweekly: import_cssweekly.cssweekly,
+  javascriptweekly: import_javascriptweekly.javascriptweekly,
+  nodeweekly: import_javascriptweekly.javascriptweekly,
+  reactstatus: import_javascriptweekly.javascriptweekly,
+  frontendfocus: import_javascriptweekly.javascriptweekly
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/types.d.ts

@@ -1,18 +1,28 @@
 import type { AnyNode, Cheerio, CheerioAPI } from 'cheerio';
 import { Item } from 'rss-parser';
+import { SnapshotResult } from './parseContent/newsletter/types';
 export type RssItem = Item & {
     source?: string;
     author?: string;
 };
-export interface ParseStrategy {
+interface ParseStrategyBase {
     parse: boolean;
     fetcher?: 'http' | 'playwright';
     getContentElementFromArticle?: (aritcle: CheerioAPI) => Cheerio<AnyNode>;
+    imageSrcAttr?: string;
+    getThumbs?: (items: Item[]) => Promise<Record<string, string> | undefined>;
+}
+export interface ParseStrategyArticle extends ParseStrategyBase {
+    articleType?: 'article';
     getContentFromHtml?: ($: CheerioAPI, node: Cheerio<AnyNode>) => string;
     getExtraItems?: ($: CheerioAPI, current: RichArticle, item: Item) => Promise<Record<string, any>>;
     ignoreProbeImage?: boolean;
-    getThumbs?: (items: Item[]) => Promise<Record<string, string> | undefined>;
 }
+export interface ParseStrategyMewsLetter extends ParseStrategyBase {
+    articleType: 'newsletter';
+    run: ($: CheerioAPI, node: Cheerio<AnyNode>) => Promise<SnapshotResult>;
+}
+export type ParseStrategy = ParseStrategyArticle | ParseStrategyMewsLetter;
 export interface SimpleArticle {
     type: 'simple';
     guid: string;
@@ -42,4 +52,9 @@ export interface RichArticle extends Omit<SimpleArticle, 'type'> {
     ssrContent: string;
     images: RichArticleImage[];
 }
-export type Article = SimpleArticle | RichArticle;
+export interface NewsletterArticle extends Omit<SimpleArticle, 'type'> {
+    type: 'newsletter';
+    newsletter: SnapshotResult;
+}
+export type Article = SimpleArticle | RichArticle | NewsletterArticle;
+export {};

package/dist/cjs/upload/index.d.ts

@@ -12,7 +12,7 @@ export declare function uploadContent({ app, getUploadApi, items, }: {
     items: Article[];
 } & ServerInfo): Promise<Article[] | {
     storePath: any;
-    type: "simple" | "rich";
+    type: "newsletter" | "simple" | "rich";
     guid: string;
     title: string;
     link: string;

package/dist/cjs/utils/arraySplit.d.ts

@@ -0,0 +1 @@
+export declare const arraySplit: <T>(arr: T[], fn: (item: T) => boolean) => T[][];

package/dist/cjs/utils/arraySplit.js

@@ -0,0 +1,42 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/arraySplit.ts
+var arraySplit_exports = {};
+__export(arraySplit_exports, {
+  arraySplit: () => arraySplit
+});
+module.exports = __toCommonJS(arraySplit_exports);
+var arraySplit = (arr, fn) => {
+  const rst = [];
+  let tmp = [];
+  arr.forEach((el, i) => {
+    if (fn(el)) {
+      rst.push(tmp);
+      tmp = [];
+    } else {
+      tmp.push(el);
+    }
+  });
+  rst.push(tmp);
+  return rst;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  arraySplit
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ns-rss-spider",
-  "version": "1.1.5",
+  "version": "1.1.8",
   "description": "",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",