dms-scrape 0.3.0 → 0.4.0

package/README.md

@@ -2,8 +2,8 @@
 
 ## Currently supported sources
 ### T1
-- [ ] 電子時報*
-- [ ] 工商時報*
+- [x] 電子時報 HTML ONLY
+- [x] 工商時報 HTML ONLY
 - [x] 經濟日報 (UDNMoney)
 
 ### T2
@@ -11,13 +11,13 @@
 - [x] 自由時報
 - [x] 聯合報
 - [ ] 聯合晚報
-- [ ] 中國時報*
-- [ ] 時報資訊
+- [x] 中國時報 HTML ONLY
+- [x] 時報資訊 HTML ONLY
 - [ ] 商業週刊
-- [ ] 壹蘋新聞網
+- [x] 壹蘋新聞網
 
 ### T3 電視台
-- [ ] 三立
+- [x] 三立
 - [ ] 華視
 - [ ] 三立新聞台
 - [ ] 東森新聞
@@ -28,7 +28,7 @@
 ### T4
 - [x] 中央社
 - [x] 太報
-- [x] ETDAY ETtoday
+- [x] ETtoday (ETDAY)
 - [ ] cw
 
 ### T5
@@ -36,16 +36,15 @@
 - [x] Cool3c
 - [x] ePrice
 - [x] Sogi 手機王
-- [ ] TechOrange*
+- [x] TechOrange HTML ONLY
 - [x] KOCPC 電腦王阿達
 - [x] TechNews 科技新報
-- [ ] TechBang T客邦
+- [x] TechBang T客邦
 - [x] XFastest minor bug
-- [ ] 鉅亨網
-- [ ] 聯合新聞網?
-- [ ] 點子生活
-- [ ] 智慧電子解決方案設計平台
-- [ ] Inside
+- [x] 鉅亨網
+- [x] 點子生活
+- [x] 智慧電子解決方案設計平台
+- [x] Inside
 - [ ] Engadget
 - [ ] 電獺少女
 - [ ] CTIMES

package/extractfrombrowser.js

@@ -0,0 +1,12 @@
+var fullHTML = document.documentElement.outerHTML;
+
+// Create a temporary DOM element to manipulate the HTML
+var tempDiv = document.createElement('div');
+tempDiv.innerHTML = fullHTML;
+
+// Remove all <script> and <style> tags
+var scriptsAndStyles = tempDiv.querySelectorAll('script, style, link');
+scriptsAndStyles.forEach(tag => tag.remove());
+
+// Print the HTML without the <script> and <style> tags
+console.log(tempDiv.innerHTML);

package/index.mjs

@@ -1,8 +1,41 @@
 import { processHTML } from "./lib/processHTML.mjs";
+import { v4 as uuid } from "uuid";
+import * as cheerio from 'cheerio'
+function checkLink(link) {
+  const linksNeedText = ["digitimes", "ctee", "chinatimes", "techorange"];
+  for (const l of linksNeedText) {
+    if (link.includes(l)) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+}
 export async function dmsScrape(type, link, html) {
-  return type == "link"
-    ? processHTML(link, await (await fetch(link)).text())
-    : type == "html"
-    ? processHTML(link, html)
-    : { error: "Must specify type", url: link };
+  var data;
+  if (type == "link" && checkLink(link) === false) {
+    let $ = cheerio.load(await (await fetch(link)).text());
+    $("script, style, link").remove();
+    $("*").each(function () {
+      let content = $(this).html();
+      content = content.replace(/\$\{/g, "");
+      $(this).html(content);
+    });
+    let cleanedHTML = $.html();
+    data = processHTML(link, cleanedHTML);
+  } else if (type == "link" && checkLink(link) === true) {
+    data = {
+      error: "Needs extension",
+      url: link,
+      id: uuid(),
+    };
+  } else if (type == "html") {
+    data = processHTML(link, html);
+  } else {
+    data = {
+      error: "Must specify type",
+      url: link,
+    };
+  }
+  return data;
 }

package/lib/array2String.mjs

@@ -0,0 +1,5 @@
+export function array2String(data) {
+    Array.isArray(data)
+      ? data.toString()
+      : data;
+}

package/lib/determineCategory.mjs

@@ -1,18 +1,18 @@
 export function determineCategory(title) {
     switch (true) {
       case title.includes("高通"):
-        return "Qualcomm相關新聞";
+        return "qualcomm";
   
       case title.includes("聯發科"):
-        return "MediaTek相關新聞";
+        return "mediatek";
   
       case title.includes("5G"):
-        return "無線通訊市場";
+        return "commu";
   
       case title.includes("OPPO") || title.includes("PC"):
-        return "智慧型手機/消費性電子產品";
+        return "phone";
   
       default:
-        return "其他業界重要訊息";
+        return "other";
     }
   }

package/lib/filterContent.mjs

@@ -1,45 +1,34 @@
 export function filterContent(inputArray) {
-  inputArray = inputArray.map((line) => line.trim());
+  const excludedStartWords = [
+    "一手掌握經濟脈動", "《你可能還想看》", "(", "不用抽", "記者", "▲", "<", "\\", "。", 
+    "此處", "還在靠社群媒體", "訂閱即同意", "＊本文開放", "（本文訊息", "相關資源", "• 更多關於：", "快加入 INSIDE"
+  ];
 
-  inputArray = inputArray.filter(
-    (item) =>
-      item !== "" &&
-      !item.startsWith("一手掌握經濟脈動") &&
-      item !== "《你可能還想看》" &&
-      !item.includes("請繼續往下閱讀...") &&
-      !item.startsWith("(") &&
-      !item.startsWith("不用抽") &&
-      !item.includes("核稿編輯") &&
-      !item.startsWith("記者") &&
-      !item.includes("圖／") &&
-      !item.startsWith("▲") &&
-      !item.startsWith("<") &&
-      !item.includes("引用來源：") &&
-      !item.startsWith("\\") &&
-      !item.includes("<h2>") &&
-      !item.includes("&nbsp;") &&
-      !item.endsWith("：") &&
-      !item.includes("訂閱手機王") &&
-      !item.includes("想快速知道") &&
-      !item.includes("資料來源：") &&
-      !item.includes("&nbsp;") &&
-      !item.includes("上傳") &&
-      !item.includes("消息來源") &&
-      !item.includes(".jpg") &&
-      !item.startsWith("(") &&
-      !item.includes("下載附件") &&
-      !item.includes("保存到相冊") &&
-      !item.includes(".png") &&
-      !item.includes("，請點選") &&
-      !item.includes("，請造訪") &&
-      !item.startsWith("。") &&
-      !item.includes("其他因素不同而產生差異") &&
-      !item.startsWith("此處") &&
-      !item.length <= 2 &&
-      !item.includes("中央社「一手新聞」 app") &&
-      !item.includes("本網站之文字、圖片及影音")
-  );
-  inputArray = inputArray.map((str) => str.replace(/<\/?[^>]+(>|$)/g, ""));
-  inputArray = inputArray.map((line) => line.trim());
-  return inputArray;
+  const excludedSubstrings = [
+    "請繼續往下閱讀...", "核稿編輯", "圖／", "引用來源：", "<h2>", "&nbsp;", "訂閱手機王", 
+    "想快速知道", "資料來源：", "上傳", "消息來源", ".jpg", "下載附件", "保存到相冊", ".png", 
+    "，請點選", "，請造訪", "其他因素不同而產生差異", "中央社「一手新聞」 app", 
+    "本網站之文字、圖片及影音", "#content>a:hover {", "color: ", "text-decoration: "
+  ];
+
+  const excludedEndsWith = ["："];
+
+  // Combined regex for general replacements
+  const regex = /<\/?[^>]+(>|$)|\s+/g;
+
+  return inputArray
+    .map((str) => str.replace(regex, " ").trim()) // Clean HTML and trim spaces
+    .filter((item) => {
+      // Filter based on start, substring, and length
+      if (
+        item === "" ||
+        excludedStartWords.some((word) => item.startsWith(word)) ||
+        excludedSubstrings.some((substring) => item.includes(substring)) ||
+        excludedEndsWith.some((end) => item.endsWith(end)) ||
+        item.length <= 2
+      ) {
+        return false;
+      }
+      return true;
+    });
 }

package/lib/processHTML.mjs

@@ -1,83 +1,51 @@
-import { LTN3C } from "../sources/T2/LTN3C.mjs";
-import { LTNEC } from "../sources/T2/LTNEC.mjs";
-import { UDN } from "../sources/T2/UDN.mjs";
-import { UDNMoney } from "../sources/T1/UDNMoney.mjs";
-import { ETDAY } from "../sources/T4/ETDAY.mjs";
-import { EPRICE } from "../sources/T5/EPRICE.mjs";
-import { COOL3C } from "../sources/T5/COOL3C.mjs";
-import { MASHDIGI } from "../sources/T5/MASHDIGI.mjs";
-import { SOGI } from "../sources/T5/SOGI.mjs";
 import * as cheerio from "cheerio";
-import { TECHORANGE } from "../sources/T5/TECHORANGE.mjs";
-import { DIGITIMES } from "../sources/T1/DIGITIMES.mjs";
-import { KOCPC } from "../sources/T5/KOCPC.mjs";
-import { XF } from "../sources/T5/XF.mjs";
-import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
-import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
-import { CNA } from "../sources/T4/CNA.mjs";
-import { v4 as uuid } from "uuid";
+
 import { filterContent } from "./filterContent.mjs";
 import { determineCategory } from "./determineCategory.mjs";
+import { sources } from "./sources.mjs";
+
 function determineSource(link) {
-  if (link.includes("3c.ltn")) {
-    return LTN3C; 
-  } else if (link.includes("ec.ltn")) {
-    return LTNEC;
-  } else if (link.includes("money.udn")) {
-    return UDNMoney;
-  } else if (link.includes("udn")) {
-    return UDN;
-  } else if (link.includes("ettoday")) {
-    return ETDAY;
-  } else if (link.includes("eprice")) {
-    return EPRICE;
-  } else if (link.includes("cool3c")) {
-    return COOL3C;
-  } else if (link.includes("mashdigi")) {
-    return MASHDIGI;
-  } else if (link.includes("sogi")) {
-    return SOGI;
-  } else if (link.includes("techorange")) {
-    return TECHORANGE;
-  } else if (link.includes("digitimes")) {
-    return DIGITIMES;
-  } else if (link.includes("kocpc")) {
-    return KOCPC;
-  } else if (link.includes("xfastest")) {
-    return XF;
-  } else if (link.includes("taisounds")) {
-    return TAISOUNDS;
-  } else if (link.includes("technews")) {
-    return TECHNEWS;
-  } else if (link.includes("cna")) {
-    return CNA;
-  } else {
-    return "unsupported";
+  for (const [key, value] of Object.entries(sources)) {
+    if (link.includes(key)) {
+      return value;
+    }
   }
+  return "unsupported";
 }
 
 export function processHTML(link, html) {
   const handlerFunction = determineSource(link);
   let scrapedContent;
-  handlerFunction != "unsupported"
-    ? ((scrapedContent = handlerFunction(cheerio.load(html))),
-      (scrapedContent["content"] = scrapedContent["content"]
-        ? filterContent(scrapedContent["content"]).join("\n\n")
-        : scrapedContent["content"]),
-      (scrapedContent["author"] = Array.isArray(scrapedContent["author"])
-        ? scrapedContent["author"].toString()
-        : scrapedContent["author"]),
-      (scrapedContent["date"] = Array.isArray(scrapedContent["date"])
-        ? scrapedContent["date"].toString()
-        : scrapedContent["date"]),
-      (scrapedContent["category"] = determineCategory(scrapedContent["title"])),
-      (scrapedContent["url"] = link),
-      (scrapedContent["content"] = scrapedContent["content"]))
-    : (scrapedContent = {
-        error: handlerFunction,
-        url: link,
-      });
+  if (handlerFunction != "unsupported") {
+    scrapedContent = handlerFunction(cheerio.load(html));
+
+    if (scrapedContent["content"]) {
+      scrapedContent["content"] = filterContent(scrapedContent["content"]).join(
+        "\n\n"
+      );
+    }
+
+    if (Array.isArray(scrapedContent["author"])) {
+      scrapedContent["author"] = scrapedContent["author"].toString();
+    }
+    if (Array.isArray(scrapedContent["date"])) {
+      scrapedContent["date"] = scrapedContent["date"].toString();
+    }
+
+    if (Array.isArray(scrapedContent["title"])) {
+      scrapedContent["title"] = scrapedContent["title"].toString();
+    }
+
+    scrapedContent["category"] = determineCategory(scrapedContent["title"]);
+    scrapedContent["url"] = link;
+  } else {
+    scrapedContent = {
+      error: handlerFunction,
+      url: link,
+      category: "other",
+    };
+  }
+
 
-  scrapedContent["id"] = uuid();
-  return scrapedContent
+  return scrapedContent;
 }

package/lib/sources.mjs

@@ -0,0 +1,52 @@
+import { LTN3C } from "../sources/T2/LTN3C.mjs";
+import { LTNEC } from "../sources/T2/LTNEC.mjs";
+import { UDN } from "../sources/T2/UDN.mjs";
+import { UDNMoney } from "../sources/T1/UDNMoney.mjs";
+import { ETDAY } from "../sources/T4/ETDAY.mjs";
+import { EPRICE } from "../sources/T5/EPRICE.mjs";
+import { COOL3C } from "../sources/T5/COOL3C.mjs";
+import { MASHDIGI } from "../sources/T5/MASHDIGI.mjs";
+import { SOGI } from "../sources/T5/SOGI.mjs";
+import { TECHORANGE } from "../sources/T5/TECHORANGE.mjs";
+import { DIGITIMES } from "../sources/T1/DIGITIMES.mjs";
+import { KOCPC } from "../sources/T5/KOCPC.mjs";
+import { XF } from "../sources/T5/XF.mjs";
+import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
+import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
+import { CNA } from "../sources/T4/CNA.mjs";
+import { CTEE } from "../sources/T1/CTEE.mjs";
+import { CHINATIMES } from "../sources/T2/CHINATIMES.mjs";
+import { NXTAPPLE } from "../sources/T2/NXTAPPLE.mjs";
+import { SETN } from "../sources/T3/SETN.mjs";
+import { TECHBANG } from "../sources/T5/TECHBANG.mjs";
+import { CNYES } from "../sources/T5/CNYES.mjs";
+import { SAYDIGI } from "../sources/T5/SAYDIGI.mjs";
+import { COMPOTECHASIA } from "../sources/T5/COMPOTECHASIA.mjs";
+import { INSIDE } from "../sources/T5/INSIDE.mjs";
+export const sources = {
+  "3c.ltn": LTN3C,
+  "ec.ltn": LTNEC,
+  "money.udn": UDNMoney,
+  "udn": UDN,
+  "ettoday": ETDAY,
+  "eprice": EPRICE,
+  "cool3c": COOL3C,
+  "mashdigi": MASHDIGI,
+  "sogi": SOGI,
+  "techorange": TECHORANGE,
+  "digitimes": DIGITIMES,
+  "kocpc": KOCPC,
+  "xfastest": XF,
+  "taisounds": TAISOUNDS,
+  "technews": TECHNEWS,
+  "cna": CNA,
+  "ctee": CTEE,
+  "chinatimes": CHINATIMES,
+  "nextapple": NXTAPPLE,
+  "setn": SETN,
+  "techbang": TECHBANG,
+  "cnyes": CNYES,
+  "saydigi": SAYDIGI,
+  "compotechasia": COMPOTECHASIA,
+  "inside": INSIDE
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dms-scrape",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "main": "index.mjs",
   "type": "module",
   "scripts": {

package/sources/T1/CTEE.mjs

@@ -1,18 +1,32 @@
-///////ctee  、綜合外電
-    // https://www.ctee.com.tw/news/20240821700211-439901
-    // https://www.ctee.com.tw/news/20240821701357-430704
-    // case link.includes("ctee"):
-    //   title = $("h1.main-title").text().trim();
-    //   date_source_author = `${$("li.publish-date time")
-    //     .text()
-    //     .replace(/\./g, "-")} / 工商時報 / ${$("span.name").text().trim()}`;
-    //   content = [];
-    //   // content?.push($('blockquote p').text())
-    //   $("article p").each((i, element) => {
-    //     const text = $(element).text().trim();
-    //     if (text) {
-    //       content?.push(text);
-    //     }
-    //   });
-    //   break;
-    //////
+export function CTEE($) {
+  const data = $.extract({
+    title: [{ selector: "h1.main-title", value: (el) => $(el).text().trim() }],
+    date: [
+      {
+        selector: "li.publish-date time",
+        value: (el, key) => {
+          const data = $(el).text().replace(/\./g, "-");
+          return data;
+        },
+      },
+    ],
+    author: [
+        { 
+            selector: "span.name",
+            value: (el) => $(el).text().trim()
+        }],
+    content: [
+      {
+        selector: "article p",
+        value: (el, key) => {
+          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
+          if ($(el).text().trim()) {
+            return $(el).text().trim();
+          }
+        },
+      },
+    ],
+  });
+  data["source"] = "工商時報";
+  return data;
+}

package/sources/T1/DIGITIMES.mjs

@@ -1,9 +1,23 @@
+
+
 export function DIGITIMES($) {
-  let data = {};
-  data["title"] = $("h1.news-title").text();
-  data["date"] = $("time").text();
+  const data = $.extract({
+    title: "h1.news-title",
+    date: [{selector: "time", value: (el) => $(el).text().replace(/(\d{4})\/(\d{2})\/(\d{2}) .*/, '$1-$2-$3')}],
+    
+    content: [
+      {
+        selector: "div#newsText.Article.clearfix.txt-20 > p",
+        value: (el, key) => {
+          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
+          if ($(el).text().trim().replace(/\t/g, '') != '') {
+            return $(el).text().trim().replace(/\t/g, '').trim();
+          }
+        },
+      },
+    ],
+  });
   data["source"] = "電子時報";
   data["author"] = $("font").first().text();
-  data["content"] = [];
   return data;
 }

package/sources/T1/UDNMoney.mjs

@@ -23,16 +23,16 @@ export function UDNMoney($) {
       {
         selector: "section.article-body__editor p",
         value: (el, key) => {
-          const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
+          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
           if (!$(el).text().trim() == "") {
             const sample = $(el)
               .text()
               .replace(/\$\(.*/s, "")
               .trim();
           
-              if (key < sampleKey) {
+              // if (key < sampleKey) {
                 return sample;
-              }
+              // }
             
           }
         },

package/sources/T2/CHINATIMES.mjs

@@ -1,19 +1,38 @@
 /////chinatimes bug time wrong
-    // https://www.chinatimes.com/realtimenews/20240820002976-260412?chdtv
-    // case link.includes("chinatimes"):
-    //   title = $("h1.article-title").text();
-    //   date_source_author =
-    //     $("span.date").first().text() +
-    //     " / " +
-    //     $("div.source").text() +
-    //     " / " +
-    //     $("div.author").text();
+// https://www.chinatimes.com/realtimenews/20240820002976-260412?chdtv
+// case link.includes("chinatimes"):
+//   title = $("h1.article-title").text();
+//   date_source_author =
+//     $("span.date").first().text() +
+//     " / " +
+//     $("div.source").text() +
+//     " / " +
+//     $("div.author").text();
 
-    //   content = [];
-    //   $("div.article-body p").each((index, element) => {
-    //     if (!$(element).text().trim() == "") {
-    //       content.push($(element).text().trim());
-    //     }
-    //   });
-    //   break;
-    //////
+//   content = [];
+//   $("div.article-body p").each((index, element) => {
+//     if (!$(element).text().trim() == "") {
+//       content.push($(element).text().trim());
+//     }
+//   });
+//   break;
+//////
+export function CHINATIMES($) {
+  const data = $.extract({
+    title: "h1.article-title",
+    date: [{ selector: "span.date", value: (el) => $(el).text().replace(/\//g, "-") }],
+    source: "div.source",
+    author: "div.author",
+    content: [
+      {
+        selector: "div.article-body p",
+        value: (el) => {
+          if ($(el).text().trim() !== "") {
+            return $(el).text().trim();
+          }
+        },
+      },
+    ],
+  });
+  return data;
+}

package/sources/T2/NXTAPPLE.mjs

@@ -0,0 +1,30 @@
+export function NXTAPPLE($) {
+  const data = $.extract({
+    title: "title",
+   
+    date: [
+      {
+        selector: "time:first",
+        value: (el) => $(el).attr("datetime").split("T")[0],
+      },
+    ],
+    author: [
+      {
+        selector: "a[href^='https://tw.nextapple.com/search/author/']",
+        value: (el) => $(el).text().trim(),
+      },
+    ],
+    content: [
+      {
+        selector: "div.post-content > p",
+        value: (el) => {
+          if ($(el).text().trim() !== "") {
+            return $(el).text().trim();
+          }
+        },
+      },
+    ],
+  });
+  data["source"] =  "壹蘋新聞網"
+  return data;
+}

package/sources/T3/SETN.mjs

@@ -0,0 +1,35 @@
+export function SETN($) {
+  const data = $.extract({
+    title: [{selector: "title", value: (el) => $(el).text().trim()}],
+
+    date: [
+      {
+        selector: "meta[name=lastmod]",
+        value: (el) => $(el).attr("content").split("T")[0],
+      },
+    ],
+    author: [
+      {
+        selector: "div p:not([style]):first",
+        value: (el) =>
+          $(el)
+            .text()
+            .match(/(?<=記者).{3}/)
+            ? $(el)
+                .text()
+                .match(/(?<=記者).{3}/)[0]
+                .trim()
+            : $(el).text(),
+      },
+    ],
+  });
+  data["source"] = "三立新聞網";
+  data["content"] = [];
+
+  $("div p:not([style])")
+    .slice(1)
+    .each(function () {
+      data["content"].push($(this).text());
+    });
+  return data;
+}