dms-scrape 0.1.3 → 0.2.0

package/.gitattributes

@@ -1,2 +1,2 @@
-# Auto detect text files and perform LF normalization
-* text=auto
+# Auto detect text files and perform LF normalization
+* text=auto

package/README.md

@@ -2,7 +2,7 @@
 
 ## Currently supported sources
 ### T1
-- [x] 電子時報*
+- [ ] 電子時報*
 - [ ] 工商時報*
 - [x] 經濟日報 (UDNMoney)
 

package/lib/{processHTML.mjs → determineSource.mjs}

@@ -15,9 +15,12 @@ import { XF } from "../sources/T5/XF.mjs";
 import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
 import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
 import { CNA } from "../sources/T4/CNA.mjs";
-
-function determineSource(link) {
-  if (link.includes("3c.ltn")) {
+import { handleUnsupported } from "../sources/handleUnsupported.mjs";
+export function determineSource(link) {
+  // if (link.includes("3c") == undefined) {
+  //   return handleUnsupported;
+  // } else
+   if (link.includes("3c.ltn")) {
     return LTN3C; // Return the corresponding function
   } else if (link.includes("ec.ltn")) {
     return LTNEC;
@@ -36,23 +39,27 @@ function determineSource(link) {
   } else if (link.includes("sogi")) {
     return SOGI;
   } else if (link.includes("techorange")) {
-    return TECHORANGE
+    return TECHORANGE;
   } else if (link.includes("digitimes")) {
-    return DIGITIMES
+    return DIGITIMES;
   } else if (link.includes("kocpc")) {
-    return KOCPC
+    return KOCPC;
   } else if (link.includes("xfastest")) {
-    return XF
+    return XF;
   } else if (link.includes("taisounds")) {
-    return TAISOUNDS
+    return TAISOUNDS;
   } else if (link.includes("technews")) {
-    return TECHNEWS
+    return TECHNEWS;
   } else if (link.includes("cna")) {
-    return CNA
+    return CNA;
+  } else {
+    return handleUnsupported(link);
   }
 }
 
-export function processHTML(link, html) {
-  const handlerFunction = determineSource(link)
-  return handlerFunction(cheerio.load(html));
-}
+// export function processHTML(link, html) {
+//   const handlerFunction = determineSource(link);
+//   return handlerFunction == handleUnsupported
+//     ? handlerFunction(cheerio.load(html))
+//     : handlerFunction(link);
+// }

package/lib/fetchAndScrape.mjs

@@ -2,17 +2,17 @@ import { justScrape } from "./justScrape.mjs";
 export async function fetchAndScrape(link) {
     var response;
     try {
-      if (
+      // if (
         // link.includes("digitimes") ||
-        link.includes("chinatimes") ||
-        link.includes("ctee") ||
-        link.includes("buzzorange")
-      ) {
-        return "Must use extension";
-      } else {
+      //   link.includes("chinatimes") ||
+      //   link.includes("ctee") ||
+      //   link.includes("buzzorange")
+      // ) {
+      //   return "Must use extension";
+      // } else {
         response = await fetch(link);
         return justScrape(link, await response.text());
-      }
+      // }
     } catch (error) {
       return error;
     }

package/lib/justScrape.mjs

@@ -1,22 +1,30 @@
-import { processHTML } from "./processHTML.mjs";
+// import { processHTML } from "./determineSource.mjs";
 import { filterContent } from "./filterContent.mjs";
 import { determineCategory } from "./determineCategory.mjs";
+import { determineSource } from "./determineSource.mjs";
 export function justScrape(link, html) {
   try {
-    var scrapedContent = processHTML(link, html);
-    scrapedContent["content"]
-      ? (scrapedContent["content"] = filterContent(scrapedContent["content"]))
-      : "";
+    const handlerFunction = determineSource(link);
+    var scrapedContent =
+      typeof handlerFunction == "function"
+        ? handlerFunction(cheerio.load(html))
+        : handlerFunction;
+    if (scrapedContent["error"] == undefined) {
+      scrapedContent["content"]
+        ? (scrapedContent["content"] = filterContent(scrapedContent["content"]))
+        : "";
+
+      Array.isArray(scrapedContent["author"])
+        ? (scrapedContent["author"] = scrapedContent["author"].toString())
+        : "";
+      Array.isArray(scrapedContent["date"])
+        ? (scrapedContent["date"] = scrapedContent["date"].toString())
+        : "";
+      scrapedContent["category"] = determineCategory(scrapedContent["title"]);
+      scrapedContent["url"] = link;
+      scrapedContent["content"] = scrapedContent["content"].join("\n\n");
+    }
 
-    Array.isArray(scrapedContent["author"])
-      ? (scrapedContent["author"] = scrapedContent["author"].toString())
-      : "";
-    Array.isArray(scrapedContent["date"])
-      ? (scrapedContent["date"] = scrapedContent["date"].toString())
-      : "";
-      scrapedContent["category"] = determineCategory(scrapedContent["title"])
-    scrapedContent["url"] = link;
-    scrapedContent['content'] = scrapedContent['content'].join('\n\n')
     return scrapedContent;
   } catch (error) {
     return error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dms-scrape",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "main": "index.mjs",
   "type": "module",
   "scripts": {

package/sources/handleUnsupported.mjs

@@ -0,0 +1,6 @@
+export function handleUnsupported(url) {
+    return {
+        error: "unsupported link",
+        url: url
+    }
+}

package/test.mjs

@@ -18,7 +18,7 @@ const testLinks = [
 
 const buggedLinks = ["https://www.eprice.com.tw/mobile/talk/4523/5811558/1/"];
 const testLink = [
-  "https://www.ettoday.net/news/20240905/2810968.htm",
+  "https://www.ctee.com.tw/news/20240805700832-431207",
 ];
 async function getTestLinks() {
   var response = [];
@@ -29,5 +29,5 @@ async function getTestLinks() {
   // return await response.text();
   return response;
 }
-const t = await getTestLinks()
-console.log(await t[0].content);
+
+console.log(await getTestLinks());