dms-scrape 0.2.1 → 0.3.1

package/.gitattributes

@@ -1,2 +1,2 @@
-# Auto detect text files and perform LF normalization
-* text=auto
+# Auto detect text files and perform LF normalization
+* text=auto

package/index.mjs

@@ -1,9 +1,8 @@
-import { fetchAndScrape } from "./lib/fetchAndScrape.mjs";
-import { justScrape } from "./lib/justScrape.mjs";
+import { processHTML } from "./lib/processHTML.mjs";
 export async function dmsScrape(type, link, html) {
   return type == "link"
-    ? await fetchAndScrape(link)
+    ? processHTML(link, await (await fetch(link)).text())
     : type == "html"
-    ? justScrape(link, html)
-    : "Must specify type";
+    ? processHTML(link, html)
+    : { error: "Must specify type", url: link };
 }

package/lib/{determineSource.mjs → processHTML.mjs}

@@ -15,13 +15,12 @@ import { XF } from "../sources/T5/XF.mjs";
 import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
 import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
 import { CNA } from "../sources/T4/CNA.mjs";
-import { handleUnsupported } from "../sources/handleUnsupported.mjs";
-export function determineSource(link) {
-  // if (link.includes("3c") == undefined) {
-  //   return handleUnsupported;
-  // } else
-   if (link.includes("3c.ltn")) {
-    return LTN3C; // Return the corresponding function
+import { v4 as uuid } from "uuid";
+import { filterContent } from "./filterContent.mjs";
+import { determineCategory } from "./determineCategory.mjs";
+function determineSource(link) {
+  if (link.includes("3c.ltn")) {
+    return LTN3C; 
   } else if (link.includes("ec.ltn")) {
     return LTNEC;
   } else if (link.includes("money.udn")) {
@@ -53,13 +52,32 @@ export function determineSource(link) {
   } else if (link.includes("cna")) {
     return CNA;
   } else {
-    return handleUnsupported(link);
+    return "unsupported";
   }
 }
 
-// export function processHTML(link, html) {
-//   const handlerFunction = determineSource(link);
-//   return handlerFunction == handleUnsupported
-//     ? handlerFunction(cheerio.load(html))
-//     : handlerFunction(link);
-// }
+export function processHTML(link, html) {
+  const handlerFunction = determineSource(link);
+  let scrapedContent;
+  handlerFunction != "unsupported"
+    ? ((scrapedContent = handlerFunction(cheerio.load(html))),
+      (scrapedContent["content"] = scrapedContent["content"]
+        ? filterContent(scrapedContent["content"]).join("\n\n")
+        : scrapedContent["content"]),
+      (scrapedContent["author"] = Array.isArray(scrapedContent["author"])
+        ? scrapedContent["author"].toString()
+        : scrapedContent["author"]),
+      (scrapedContent["date"] = Array.isArray(scrapedContent["date"])
+        ? scrapedContent["date"].toString()
+        : scrapedContent["date"]),
+      (scrapedContent["category"] = determineCategory(scrapedContent["title"])),
+      (scrapedContent["url"] = link),
+      (scrapedContent["content"] = scrapedContent["content"]))
+    : (scrapedContent = {
+        error: handlerFunction,
+        url: link,
+      });
+
+  scrapedContent["id"] = uuid();
+  return scrapedContent
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dms-scrape",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "main": "index.mjs",
   "type": "module",
   "scripts": {
@@ -13,6 +13,7 @@
     "esbuild": "^0.23.1"
   },
   "dependencies": {
-    "cheerio": "^1.0.0"
+    "cheerio": "^1.0.0",
+    "uuid": "^10.0.0"
   }
 }

package/sources/T1/UDNMoney.mjs

@@ -23,16 +23,16 @@ export function UDNMoney($) {
       {
         selector: "section.article-body__editor p",
         value: (el, key) => {
-          const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
+          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
           if (!$(el).text().trim() == "") {
             const sample = $(el)
               .text()
               .replace(/\$\(.*/s, "")
               .trim();
           
-              if (key < sampleKey) {
+              // if (key < sampleKey) {
                 return sample;
-              }
+              // }
             
           }
         },