dms-scrape 0.1.0 → 0.1.2

package/.vscode/launch.json

@@ -12,7 +12,7 @@
             "skipFiles": [
                 "<node_internals>/**"
             ],
-            "program": "${workspaceFolder}\\index.mjs"
+            "program": "${workspaceFolder}\\test.mjs"
         }
     ]
 }

package/README.md

@@ -2,45 +2,64 @@
 
 ## Currently supported sources
 ### T1
-- [x] 電子*
-- [ ] 工商*
-- [x] 經濟 (UDNMoney)
+- [x] 電子時報*
+- [ ] 工商時報*
+- [x] 經濟日報 (UDNMoney)
+
 ### T2
-- [x] 自由 3C
-- [x] 自由
-- [x] 聯合
-- [ ] 中國*
+- [x] 自由時報 3C
+- [x] 自由時報
+- [x] 聯合報
+- [ ] 聯合晚報
+- [ ] 中國時報*
+- [ ] 時報資訊
+- [ ] 商業週刊
+- [ ] 壹蘋新聞網
+
 ### T3 電視台
-- [ ] ETtoday
 - [ ] 三立
+- [ ] 華視
+- [ ] 三立新聞台
+- [ ] 東森新聞
+- [ ] 民視新聞網
+- [ ] TVBS
+- [ ] 非凡新聞
+
 ### T4
-- [ ] 中央社
-- [ ] 太報
+- [x] 中央社
+- [x] 太報
+- [x] ETDAY ETtoday
+- [ ] cw
+
 ### T5
 - [x] Mashdigi
 - [x] Cool3c
 - [x] ePrice
-- [x] Sogi
+- [x] Sogi 手機王
 - [ ] TechOrange*
-- [x] KOCPC
-- [x] TechNews
-- [ ] TechBang
-- [ ] XFastest
-### T6 財經媒體 
-
+- [x] KOCPC 電腦王阿達
+- [x] TechNews 科技新報
+- [ ] TechBang T客邦
+- [x] XFastest minor bug
+- [ ] 鉅亨網
+- [ ] 聯合新聞網?
+- [ ] 點子生活
+- [ ] 智慧電子解決方案設計平台
+- [ ] Inside
+- [ ] Engadget
+- [ ] 電獺少女
+- [ ] CTIMES
+- [ ] 新唐人亞太台
+- [ ] 新通訊元件雜誌
+- [ ] 中時電子報
+- [ ] 新電子
+- [ ] CTWant (陳柔蓁)
+- [ ] 數位時代
+- [ ] lpcomment
+- [ ] 2cm
+- [ ] techudn
 
-## To be supported
-- nextapple
-- cna
-- setn
-- bnext
-- saydigi
-- inside
-- lpcomment
-- techudnc
-- ctimes
-- ctwant
-- cnyes
-- moneydj
-- 2cm
-- investor
+### T6 財經媒體 
+- [ ] 財訊雙週刊
+- [ ] 財訊快報
+- [ ] MoneyDJ

package/index.mjs

@@ -7,9 +7,3 @@ export async function dmsScrape(type, link, html) {
     ? justScrape(link, html)
     : "Must specify type";
 }
-console.log(
-  await dmsScrape(
-    "link",
-    "https://technews.tw/2024/09/12/snapdragon-8-gen-4-and-dimensity-9400-configuration-unveiled/"
-  )
-);

package/lib/determineCategory.mjs

@@ -0,0 +1,18 @@
+export function determineCategory(title) {
+    switch (true) {
+      case title.includes("高通"):
+        return "Qualcomm相關新聞";
+  
+      case title.includes("聯發科"):
+        return "MediaTek相關新聞";
+  
+      case title.includes("5G"):
+        return "無線通訊市場";
+  
+      case title.includes("OPPO") || title.includes("PC"):
+        return "智慧型手機/消費性電子產品";
+  
+      default:
+        return "其他業界重要訊息";
+    }
+  }

package/lib/filterContent.mjs

@@ -8,7 +8,6 @@ export function filterContent(inputArray) {
       item !== "《你可能還想看》" &&
       !item.includes("請繼續往下閱讀...") &&
       !item.startsWith("(") &&
-      !item.startsWith("（") &&
       !item.startsWith("不用抽") &&
       !item.includes("核稿編輯") &&
       !item.startsWith("記者") &&
@@ -25,12 +24,20 @@ export function filterContent(inputArray) {
       !item.includes("資料來源：") &&
       !item.includes(" ") &&
       !item.includes("上傳") &&
-      !item.includes("消息來源") && 
-      !item.includes(".jpg") && 
+      !item.includes("消息來源") &&
+      !item.includes(".jpg") &&
       !item.startsWith("(") &&
       !item.includes("下載附件") &&
       !item.includes("保存到相冊") &&
-      !item.includes(".png")
+      !item.includes(".png") &&
+      !item.includes("，請點選") &&
+      !item.includes("，請造訪") &&
+      !item.startsWith("。") &&
+      !item.includes("其他因素不同而產生差異") &&
+      !item.startsWith("此處") &&
+      !item.length <= 2 &&
+      !item.includes("中央社「一手新聞」 app") &&
+      !item.includes("本網站之文字、圖片及影音")
   );
   inputArray = inputArray.map((str) => str.replace(/<\/?[^>]+(>|$)/g, ""));
   inputArray = inputArray.map((line) => line.trim());

package/lib/justScrape.mjs

@@ -1,14 +1,23 @@
 import { processHTML } from "./processHTML.mjs";
 import { filterContent } from "./filterContent.mjs";
+import { determineCategory } from "./determineCategory.mjs";
 export function justScrape(link, html) {
-    try {
-      var scrapedContent = processHTML(link, html)
-      scrapedContent["content"] = scrapedContent["content"]
-        ? filterContent(scrapedContent["content"])
-        : "";
-      scrapedContent["url"] = link;
-      return scrapedContent;
-    } catch (error) {
-      return error;
-    }
-  }
+  try {
+    var scrapedContent = processHTML(link, html);
+    scrapedContent["content"]
+      ? (scrapedContent["content"] = filterContent(scrapedContent["content"]))
+      : "";
+
+    Array.isArray(scrapedContent["author"])
+      ? (scrapedContent["author"] = scrapedContent["author"].toString())
+      : "";
+    Array.isArray(scrapedContent["date"])
+      ? (scrapedContent["date"] = scrapedContent["date"].toString())
+      : "";
+      scrapedContent["category"] = determineCategory(scrapedContent["title"])
+    scrapedContent["url"] = link;
+    return scrapedContent;
+  } catch (error) {
+    return error;
+  }
+}

package/lib/processHTML.mjs

@@ -2,7 +2,7 @@ import { LTN3C } from "../sources/T2/LTN3C.mjs";
 import { LTNEC } from "../sources/T2/LTNEC.mjs";
 import { UDN } from "../sources/T2/UDN.mjs";
 import { UDNMoney } from "../sources/T1/UDNMoney.mjs";
-import { ETDAY } from "../sources/T3/ETDAY.mjs";
+import { ETDAY } from "../sources/T4/ETDAY.mjs";
 import { EPRICE } from "../sources/T5/EPRICE.mjs";
 import { COOL3C } from "../sources/T5/COOL3C.mjs";
 import { MASHDIGI } from "../sources/T5/MASHDIGI.mjs";
@@ -12,8 +12,9 @@ import { TECHORANGE } from "../sources/T5/TECHORANGE.mjs";
 import { DIGITIMES } from "../sources/T1/DIGITIMES.mjs";
 import { KOCPC } from "../sources/T5/KOCPC.mjs";
 import { XF } from "../sources/T5/XF.mjs";
-import { TAISOUNDS } from "../sources/T4/TAISOUNDS(unfinished).mjs";
+import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
 import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
+import { CNA } from "../sources/T4/CNA.mjs";
 
 function determineSource(link) {
   if (link.includes("3c.ltn")) {
@@ -46,6 +47,8 @@ function determineSource(link) {
     return TAISOUNDS
   } else if (link.includes("technews")) {
     return TECHNEWS
+  } else if (link.includes("cna")) {
+    return CNA
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dms-scrape",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "main": "index.mjs",
   "type": "module",
   "scripts": {

package/sources/T1/UDNMoney.mjs

@@ -1,7 +1,6 @@
-
 export function UDNMoney($) {
   const data = $.extract({
-    headline: "h1#story_art_title",
+    title: "h1#story_art_title",
     date: [
       {
         selector: "time.article-body__time",
@@ -24,11 +23,17 @@ export function UDNMoney($) {
       {
         selector: "section.article-body__editor p",
         value: (el, key) => {
+          const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
           if (!$(el).text().trim() == "") {
-            return $(el)
+            const sample = $(el)
               .text()
               .replace(/\$\(.*/s, "")
               .trim();
+          
+              if (key < sampleKey) {
+                return sample;
+              }
+            
           }
         },
       },
@@ -37,3 +42,5 @@ export function UDNMoney($) {
   data["source"] = "經濟日報";
   return data;
 }
+
+//minor unresolved additional headlines

package/sources/T2/LTN3C.mjs

@@ -1,6 +1,6 @@
 export function LTN3C($) {
   const data = $.extract({
-    headline: "div.whitecon.borderline.boxTitle.boxText h1",
+    title: "div.whitecon.borderline.boxTitle.boxText h1",
     date: [
       {
         selector: "span.time",
@@ -29,5 +29,6 @@ export function LTN3C($) {
       },
     ],
   });
+  data["source"] = "自由時報"
   return data;
 }

package/sources/T2/LTNEC.mjs

@@ -1,6 +1,6 @@
 export function LTNEC($) {
   const data = $.extract({
-    headline: "div.whitecon.boxTitle.boxText h1",
+    title: "div.whitecon.boxTitle.boxText h1",
     date: [
       {
         selector: "span.time:nth-of-type(1)",
@@ -35,6 +35,6 @@ export function LTNEC($) {
       },
     ],
   });
-
+data["source"] = "自由時報"
   return data;
 }

package/sources/T2/UDN.mjs

@@ -1,12 +1,11 @@
 export function UDN($) {
   const data = $.extract({
-    headline: "h1.article-content__title",
+    title: "h1.article-content__title",
     date: [
       {
         selector: "time.article-content__time",
         value: (el, key) => {
-          const data = $(el).text().replace(/\//g, "-")
-          .slice(0, 10);
+          const data = $(el).text().replace(/\//g, "-").slice(0, 10);
           return data;
         },
       },
@@ -16,14 +15,12 @@ export function UDN($) {
       {
         selector: "section.article-content__editor  p",
         value: (el, key) => {
-          const text = $(el)
-            .text()
-            .trim();
+          const text = $(el).text().trim();
           return text;
         },
       },
     ],
   });
-
+  data["source"] = "聯合報";
   return data;
 }

package/sources/T4/CNA.mjs

@@ -0,0 +1,33 @@
+export function CNA($) {
+  const data = $.extract({
+    content: [
+      {
+        selector: "div.paragraph > p",
+        value: (el, key) => {
+          return $(el).text();
+        },
+      },
+    ],
+    author: [
+      {
+        selector: "div.paragraph > p",
+        value: (el, key) => {
+          if ( $(el).text().includes("（中央社")) {
+            let index = $(el).text().indexOf("記者");
+
+            // Extract the 3 characters after "記者"
+            return $(el)
+              .text()
+              .substring(index + 2, index + 5);
+            // return index
+          }
+        },
+      },
+    ],
+  });
+  data["date"] = $('meta[property="article:published_time"]')
+    .attr("content")
+    .slice(0, 10);
+  data["source"] = "中央社";
+  return data;
+}

package/sources/{T3 → T4}/ETDAY.mjs

@@ -1,9 +1,7 @@
-import * as cheerio from "cheerio";
-export function ETDAY(html) {
-  const $ = cheerio.load(html);
+export function ETDAY($) {
   const data = $.extract({
-    headline: "h1.title",
-    time: [
+    title: "h1.title",
+    date: [
       {
         selector: "time.date",
         value: (el, key) => {

package/sources/T4/TAISOUNDS.mjs

@@ -0,0 +1,21 @@
+export function TAISOUNDS($) {
+  const data = $.extract({
+    date: [
+      {
+        selector: "div.publish",
+        value: (el, key) => {
+          const eachValue = $(el).text().trim();
+          if (eachValue !== "") {
+            return eachValue.slice(0, 10);
+          }
+        },
+      },
+    ],
+    
+  });
+  data["title"] = $("div.news-box h1").text();
+  data["source"] = "太報";
+  data["author"] = $("a[href^='/more/reporternews']").text();
+  data["content"] = $("div.news-box-text.border").html().split("\n").filter((item) => item.includes("<br>")).toString().split(/<\/?[^>]+>/).filter((item)=> item != '')
+  return data;
+}

package/test.mjs

@@ -0,0 +1,33 @@
+import { dmsScrape } from "./index.mjs";
+
+const testLinks = [
+  "https://www.cool3c.com/article/202079",
+
+  "https://www.kocpc.com.tw/archives/563640",
+  "https://mashdigi.com/qualcomm-continues-to-launch-the-snapdragon-x-plus-processor-composed-of-8-sets-of-performance-cores-expanding-the-copilot-pc-product-lineup/",
+  "https://www.sogi.com.tw/articles/samsung_galaxy_s25_ultra/6262880",
+  "https://technews.tw/2024/09/12/snapdragon-8-gen-4-and-dimensity-9400-configuration-unveiled/",
+  "https://www.xfastest.com/thread-291944-1-1.html",
+  "https://www.ettoday.net/news/20240905/2810968.htm",
+  "https://udn.com/news/story/7240/8145957",
+  "https://money.udn.com/money/story/5613/8231957",
+  "https://3c.ltn.com.tw/news/59270",
+  "https://ec.ltn.com.tw/article/breakingnews/4790282",
+  "https://www.digitimes.com.tw/tech/dt/n/shwnws.asp?id=0000700414_DTZ3HLET2I9D7N38H02NN",
+];
+
+const buggedLinks = ["https://www.eprice.com.tw/mobile/talk/4523/5811558/1/"];
+const testLink = [
+  "https://www.ettoday.net/news/20240905/2810968.htm",
+];
+async function getTestLinks() {
+  var response = [];
+  for (const link of testLink) {
+    response.push(await dmsScrape("link", link));
+    // response = await fetch(link);
+  }
+  // return await response.text();
+  return response;
+}
+
+console.log(await getTestLinks());

package/sources/T4/TAISOUNDS(unfinished).mjs

@@ -1,9 +0,0 @@
-export function TAISOUNDS($) {
-  let data = {};
-  data["title"] = $("div.news-box h1").text();
-  data["date"] = "";
-  data["source"] = "太報";
-  data["author"] = $("a[href^='/more/reporternews']").text();
-  data["content"] = [];
-  return data;
-}