dms-scrape 0.4.0 → 0.4.1

package/.gitattributes

@@ -1,2 +1,2 @@
-# Auto detect text files and perform LF normalization
-* text=auto
+# Auto detect text files and perform LF normalization
+* text=auto

package/README.md

@@ -19,7 +19,6 @@
 ### T3 電視台
 - [x] 三立
 - [ ] 華視
-- [ ] 三立新聞台
 - [ ] 東森新聞
 - [ ] 民視新聞網
 - [ ] TVBS

package/extractfrombrowser.js

@@ -5,7 +5,7 @@ var tempDiv = document.createElement('div');
 tempDiv.innerHTML = fullHTML;
 
 // Remove all <script> and <style> tags
-var scriptsAndStyles = tempDiv.querySelectorAll('script, style, link');
+var scriptsAndStyles = tempDiv.querySelectorAll('script, style, link, g, noscript, svg, img, symbol, figure, figcaption, ins');
 scriptsAndStyles.forEach(tag => tag.remove());
 
 // Print the HTML without the <script> and <style> tags

package/index.mjs

@@ -1,15 +1,23 @@
 import { processHTML } from "./lib/processHTML.mjs";
 import { v4 as uuid } from "uuid";
-import * as cheerio from 'cheerio'
+import * as cheerio from "cheerio";
 function checkLink(link) {
-  const linksNeedText = ["digitimes", "ctee", "chinatimes", "techorange"];
+  let resp;
+  const linksNeedText = [
+    "https://www.digitimes",
+    "https://www.ctee",
+    "https://www.chinatimes",
+    "https://buzzorange.com/techorange",
+  ];
   for (const l of linksNeedText) {
-    if (link.includes(l)) {
-      return true;
+    if (link.startsWith(l)) {
+      resp = true;
+      break;
     } else {
-      return false;
+      resp = false;
     }
   }
+  return resp;
 }
 export async function dmsScrape(type, link, html) {
   var data;

package/lib/determineCategory.mjs

@@ -1,15 +1,15 @@
 export function determineCategory(title) {
     switch (true) {
-      case title.includes("高通"):
+      case title?.includes("高通"):
         return "qualcomm";
   
-      case title.includes("聯發科"):
+      case title?.includes("聯發科"):
         return "mediatek";
   
-      case title.includes("5G"):
+      case title?.includes("5G"):
         return "commu";
   
-      case title.includes("OPPO") || title.includes("PC"):
+      case title?.includes("OPPO") || title?.includes("PC"):
         return "phone";
   
       default:

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "dms-scrape",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "main": "index.mjs",
   "type": "module",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node test.mjs"
   },
   "author": "",
   "license": "ISC",

package/sources/T2/CHINATIMES.mjs

@@ -20,9 +20,14 @@
 export function CHINATIMES($) {
   const data = $.extract({
     title: "h1.article-title",
-    date: [{ selector: "span.date", value: (el) => $(el).text().replace(/\//g, "-") }],
-    source: "div.source",
-    author: "div.author",
+    date: [
+      {
+        selector: "span.date:first",
+        value: (el) => $(el).text().replace(/\//g, "-"),
+      },
+    ],
+    source: [{ selector: "div.source", value: (el) => $(el).text().trim() }],
+    author: [{ selector: "div.author", value: (el) => $(el).text().trim() }],
     content: [
       {
         selector: "div.article-body p",