dms-scrape 0.4.4 → 0.4.6

package/package.json

@@ -1,14 +1,18 @@
 {
   "name": "dms-scrape",
-  "version": "0.4.4",
-  "main": "index.mjs",
+  "version": "0.4.6",
+  "main": "dist/index.js",
   "type": "module",
   "scripts": {
-    "test": "node test.mjs"
+    "test": "node test.mjs",
+    "build": "node build.mjs"
   },
-  "author": "",
+  "files": [
+    "dist/*"
+  ],
+  "author": "lostmypillow",
   "license": "ISC",
-  "description": "",
+  "description": "Scraper for CompassPR DMS Scraper",
   "devDependencies": {
     "esbuild": "^0.23.1"
   },

package/.gitattributes

@@ -1,2 +0,0 @@
-# Auto detect text files and perform LF normalization
-* text=auto

package/.vscode/launch.json

@@ -1,18 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Launch Program",
-            "console": "integratedTerminal",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "program": "${workspaceFolder}\\test.mjs"
-        }
-    ]
-}

package/dms-scrape.d.ts

@@ -1 +0,0 @@
-declare module 'dms-scrape'

package/extractfrombrowser.js

@@ -1,12 +0,0 @@
-var fullHTML = document.documentElement.outerHTML;
-
-// Create a temporary DOM element to manipulate the HTML
-var tempDiv = document.createElement('div');
-tempDiv.innerHTML = fullHTML;
-
-// Remove all <script> and <style> tags
-var scriptsAndStyles = tempDiv.querySelectorAll('script, style, link, g, noscript, svg, img, symbol, figure, figcaption, ins');
-scriptsAndStyles.forEach(tag => tag.remove());
-
-// Print the HTML without the <script> and <style> tags
-console.log(tempDiv.innerHTML);

package/index.mjs

@@ -1,50 +0,0 @@
-import { processHTML } from "./lib/processHTML.mjs";
-import { v4 as uuid } from "uuid";
-import * as cheerio from "cheerio";
-function checkLink(link) {
-  let resp;
-  const linksNeedText = [
-    "https://www.digitimes",
-    "https://www.ctee",
-    "https://www.chinatimes",
-    "https://buzzorange.com/techorange",
-  ];
-  for (const l of linksNeedText) {
-    if (link.startsWith(l)) {
-      resp = true;
-      break;
-    } else {
-      resp = false;
-    }
-  }
-  return resp;
-}
-export async function dmsScrape(type, link, html) {
-  var data;
-  if (type == "link" && checkLink(link) === false) {
-    let $ = cheerio.load(await (await fetch(link)).text());
-    $("script, style, link").remove();
-    $("*").each(function () {
-      let content = $(this).html();
-      content = content.replace(/\$\{/g, "");
-      $(this).html(content);
-    });
-    let cleanedHTML = $.html();
-    data = processHTML(link, cleanedHTML);
-  } else if (type == "link" && checkLink(link) === true) {
-    data = {
-      error: "Needs extension",
-      url: link,
-      id: uuid(),
-      category: "other"
-    };
-  } else if (type == "html") {
-    data = processHTML(link, html);
-  } else {
-    data = {
-      error: "Must specify type",
-      url: link,
-    };
-  }
-  return data;
-}

package/lib/array2String.mjs

@@ -1,5 +0,0 @@
-export function array2String(data) {
-    Array.isArray(data)
-      ? data.toString()
-      : data;
-}

package/lib/determineCategory.mjs

@@ -1,18 +0,0 @@
-export function determineCategory(title) {
-    switch (true) {
-      case title?.includes("高通"):
-        return "qualcomm";
-  
-      case title?.includes("聯發科"):
-        return "mediatek";
-  
-      case title?.includes("5G"):
-        return "commu";
-  
-      case title?.includes("OPPO") || title?.includes("PC"):
-        return "phone";
-  
-      default:
-        return "other";
-    }
-  }

package/lib/filterContent.mjs

@@ -1,34 +0,0 @@
-export function filterContent(inputArray) {
-  const excludedStartWords = [
-    "一手掌握經濟脈動", "《你可能還想看》", "(", "不用抽", "記者", "▲", "<", "\\", "。", 
-    "此處", "還在靠社群媒體", "訂閱即同意", "＊本文開放", "（本文訊息", "相關資源", "• 更多關於：", "快加入 INSIDE"
-  ];
-
-  const excludedSubstrings = [
-    "請繼續往下閱讀...", "核稿編輯", "圖／", "引用來源：", "<h2>", "&nbsp;", "訂閱手機王", 
-    "想快速知道", "資料來源：", "上傳", "消息來源", ".jpg", "下載附件", "保存到相冊", ".png", 
-    "，請點選", "，請造訪", "其他因素不同而產生差異", "中央社「一手新聞」 app", 
-    "本網站之文字、圖片及影音", "#content>a:hover {", "color: ", "text-decoration: "
-  ];
-
-  const excludedEndsWith = ["："];
-
-  // Combined regex for general replacements
-  const regex = /<\/?[^>]+(>|$)|\s+/g;
-
-  return inputArray
-    .map((str) => str.replace(regex, " ").trim()) // Clean HTML and trim spaces
-    .filter((item) => {
-      // Filter based on start, substring, and length
-      if (
-        item === "" ||
-        excludedStartWords.some((word) => item.startsWith(word)) ||
-        excludedSubstrings.some((substring) => item.includes(substring)) ||
-        excludedEndsWith.some((end) => item.endsWith(end)) ||
-        item.length <= 2
-      ) {
-        return false;
-      }
-      return true;
-    });
-}

package/lib/processHTML.mjs

@@ -1,50 +0,0 @@
-import * as cheerio from "cheerio";
-
-import { filterContent } from "./filterContent.mjs";
-import { determineCategory } from "./determineCategory.mjs";
-import { sources } from "./sources.mjs";
-
-function determineSource(link) {
-  for (const [key, value] of Object.entries(sources)) {
-    if (link.includes(key)) {
-      return value;
-    }
-  }
-  return "unsupported";
-}
-
-export function processHTML(link, html) {
-  const handlerFunction = determineSource(link);
-  let scrapedContent;
-  if (handlerFunction != "unsupported") {
-    scrapedContent = handlerFunction(cheerio.load(html));
-
-    if (scrapedContent["content"]) {
-      scrapedContent["content"] = filterContent(scrapedContent["content"]).join(
-        "\n\n"
-      );
-    }
-
-    if (Array.isArray(scrapedContent["author"])) {
-      scrapedContent["author"] = scrapedContent["author"].toString();
-    }
-    if (Array.isArray(scrapedContent["date"])) {
-      scrapedContent["date"] = scrapedContent["date"].toString();
-    }
-
-    if (Array.isArray(scrapedContent["title"])) {
-      scrapedContent["title"] = scrapedContent["title"].toString();
-    }
-
-    scrapedContent["category"] = determineCategory(scrapedContent["title"]);
-    scrapedContent["url"] = link;
-  } else {
-    scrapedContent = {
-      error: handlerFunction,
-      url: link,
-    };
-  }
-
-
-  return scrapedContent;
-}

package/lib/sources.mjs

@@ -1,52 +0,0 @@
-import { LTN3C } from "../sources/T2/LTN3C.mjs";
-import { LTNEC } from "../sources/T2/LTNEC.mjs";
-import { UDN } from "../sources/T2/UDN.mjs";
-import { UDNMoney } from "../sources/T1/UDNMoney.mjs";
-import { ETDAY } from "../sources/T4/ETDAY.mjs";
-import { EPRICE } from "../sources/T5/EPRICE.mjs";
-import { COOL3C } from "../sources/T5/COOL3C.mjs";
-import { MASHDIGI } from "../sources/T5/MASHDIGI.mjs";
-import { SOGI } from "../sources/T5/SOGI.mjs";
-import { TECHORANGE } from "../sources/T5/TECHORANGE.mjs";
-import { DIGITIMES } from "../sources/T1/DIGITIMES.mjs";
-import { KOCPC } from "../sources/T5/KOCPC.mjs";
-import { XF } from "../sources/T5/XF.mjs";
-import { TAISOUNDS } from "../sources/T4/TAISOUNDS.mjs";
-import { TECHNEWS } from "../sources/T5/TECHNEWS.mjs";
-import { CNA } from "../sources/T4/CNA.mjs";
-import { CTEE } from "../sources/T1/CTEE.mjs";
-import { CHINATIMES } from "../sources/T2/CHINATIMES.mjs";
-import { NXTAPPLE } from "../sources/T2/NXTAPPLE.mjs";
-import { SETN } from "../sources/T3/SETN.mjs";
-import { TECHBANG } from "../sources/T5/TECHBANG.mjs";
-import { CNYES } from "../sources/T5/CNYES.mjs";
-import { SAYDIGI } from "../sources/T5/SAYDIGI.mjs";
-import { COMPOTECHASIA } from "../sources/T5/COMPOTECHASIA.mjs";
-import { INSIDE } from "../sources/T5/INSIDE.mjs";
-export const sources = {
-  "3c.ltn": LTN3C,
-  "ec.ltn": LTNEC,
-  "money.udn": UDNMoney,
-  "udn": UDN,
-  "ettoday": ETDAY,
-  "eprice": EPRICE,
-  "cool3c": COOL3C,
-  "mashdigi": MASHDIGI,
-  "sogi": SOGI,
-  "techorange": TECHORANGE,
-  "digitimes": DIGITIMES,
-  "kocpc": KOCPC,
-  "xfastest": XF,
-  "taisounds": TAISOUNDS,
-  "technews": TECHNEWS,
-  "cna": CNA,
-  "ctee": CTEE,
-  "chinatimes": CHINATIMES,
-  "nextapple": NXTAPPLE,
-  "setn": SETN,
-  "techbang": TECHBANG,
-  "cnyes": CNYES,
-  "saydigi": SAYDIGI,
-  "compotechasia": COMPOTECHASIA,
-  "inside": INSIDE
-};

package/sources/T1/CTEE.mjs

@@ -1,32 +0,0 @@
-export function CTEE($) {
-  const data = $.extract({
-    title: [{ selector: "h1.main-title", value: (el) => $(el).text().trim() }],
-    date: [
-      {
-        selector: "li.publish-date time",
-        value: (el, key) => {
-          const data = $(el).text().replace(/\./g, "-");
-          return data;
-        },
-      },
-    ],
-    author: [
-        { 
-            selector: "span.name",
-            value: (el) => $(el).text().trim()
-        }],
-    content: [
-      {
-        selector: "article p",
-        value: (el, key) => {
-          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
-          if ($(el).text().trim()) {
-            return $(el).text().trim();
-          }
-        },
-      },
-    ],
-  });
-  data["source"] = "工商時報";
-  return data;
-}

package/sources/T1/DIGITIMES.mjs

@@ -1,23 +0,0 @@
-
-
-export function DIGITIMES($) {
-  const data = $.extract({
-    title: "h1.news-title",
-    date: [{selector: "time", value: (el) => $(el).text().replace(/(\d{4})\/(\d{2})\/(\d{2}) .*/, '$1-$2-$3')}],
-    
-    content: [
-      {
-        selector: "div#newsText.Article.clearfix.txt-20 > p",
-        value: (el, key) => {
-          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
-          if ($(el).text().trim().replace(/\t/g, '') != '') {
-            return $(el).text().trim().replace(/\t/g, '').trim();
-          }
-        },
-      },
-    ],
-  });
-  data["source"] = "電子時報";
-  data["author"] = $("font").first().text();
-  return data;
-}

package/sources/T1/UDNMoney.mjs

@@ -1,46 +0,0 @@
-export function UDNMoney($) {
-  const data = $.extract({
-    title: "h1#story_art_title",
-    date: [
-      {
-        selector: "time.article-body__time",
-        value: (el, key) => {
-          const data = $(el).text().replace(/\//g, "-").slice(0, 10);
-          return data;
-        },
-      },
-    ],
-    author: [
-      {
-        selector: "div.article-body__info > span",
-        value: (el, key) => {
-          const text = $(el).text();
-          return text.replace(/.*記者(.{3}).*/, "$1");
-        },
-      },
-    ],
-    content: [
-      {
-        selector: "section.article-body__editor p",
-        value: (el, key) => {
-          // const sampleKey = this.findIndex(item  => item.includes("延伸閱讀"))
-          if (!$(el).text().trim() == "") {
-            const sample = $(el)
-              .text()
-              .replace(/\$\(.*/s, "")
-              .trim();
-          
-              // if (key < sampleKey) {
-                return sample;
-              // }
-            
-          }
-        },
-      },
-    ],
-  });
-  data["source"] = "經濟日報";
-  return data;
-}
-
-//minor unresolved additional headlines

package/sources/T2/CHINATIMES.mjs

@@ -1,43 +0,0 @@
-/////chinatimes bug time wrong
-// https://www.chinatimes.com/realtimenews/20240820002976-260412?chdtv
-// case link.includes("chinatimes"):
-//   title = $("h1.article-title").text();
-//   date_source_author =
-//     $("span.date").first().text() +
-//     " / " +
-//     $("div.source").text() +
-//     " / " +
-//     $("div.author").text();
-
-//   content = [];
-//   $("div.article-body p").each((index, element) => {
-//     if (!$(element).text().trim() == "") {
-//       content.push($(element).text().trim());
-//     }
-//   });
-//   break;
-//////
-export function CHINATIMES($) {
-  const data = $.extract({
-    title: "h1.article-title",
-    date: [
-      {
-        selector: "span.date:first",
-        value: (el) => $(el).text().replace(/\//g, "-"),
-      },
-    ],
-    source: { selector: "div.source", value: (el) => $(el).text().trim() },
-    author: [{ selector: "div.author", value: (el) => $(el).text().trim() }],
-    content: [
-      {
-        selector: "div.article-body p",
-        value: (el) => {
-          if ($(el).text().trim() !== "") {
-            return $(el).text().trim();
-          }
-        },
-      },
-    ],
-  });
-  return data;
-}

package/sources/T2/LTN3C.mjs

@@ -1,34 +0,0 @@
-export function LTN3C($) {
-  const data = $.extract({
-    title: "div.whitecon.borderline.boxTitle.boxText h1",
-    date: [
-      {
-        selector: "span.time",
-        value: (el, key) => {
-          const data = $(el).text().split(" ").slice(0)[0].replace(/\//g, "-");
-          return data;
-        },
-      },
-    ],
-    author: [
-      {
-        selector: "span.author",
-        value: (el) => {
-          const data = $(el).text().split("").slice(-3).join("");
-          return data;
-        },
-      },
-    ],
-    content: [
-      {
-        selector: "div.text p",
-        value: (el, key) => {
-          const text = $(el).text().trim();
-          return text;
-        },
-      },
-    ],
-  });
-  data["source"] = "自由時報"
-  return data;
-}

package/sources/T2/LTNEC.mjs

@@ -1,40 +0,0 @@
-export function LTNEC($) {
-  const data = $.extract({
-    title: "div.whitecon.boxTitle.boxText h1",
-    date: [
-      {
-        selector: "span.time:nth-of-type(1)",
-        value: (el, key) => {
-          const data = $(el).text().split(" ").slice(0)[0].replace(/\//g, "-");
-          if (data != "") {
-            return data;
-          }
-          
-        },
-      },
-    ],
-    author: [
-      {
-        selector: "div.text p:nth-of-type(2)",
-        value: (el) => {
-          const data = $(el).text().split("／")[0];
-          return data;
-        },
-      },
-    ],
-    content: [
-      {
-        selector: "div.text p",
-        value: (el, key) => {
-          const text = $(el)
-            .text()
-            .trim()
-            .replace(/〔.*?〕/g, "");
-          return text;
-        },
-      },
-    ],
-  });
-data["source"] = "自由時報"
-  return data;
-}

package/sources/T2/NXTAPPLE.mjs

@@ -1,30 +0,0 @@
-export function NXTAPPLE($) {
-  const data = $.extract({
-    title: "title",
-   
-    date: [
-      {
-        selector: "time:first",
-        value: (el) => $(el).attr("datetime").split("T")[0],
-      },
-    ],
-    author: [
-      {
-        selector: "a[href^='https://tw.nextapple.com/search/author/']",
-        value: (el) => $(el).text().trim(),
-      },
-    ],
-    content: [
-      {
-        selector: "div.post-content > p",
-        value: (el) => {
-          if ($(el).text().trim() !== "") {
-            return $(el).text().trim();
-          }
-        },
-      },
-    ],
-  });
-  data["source"] =  "壹蘋新聞網"
-  return data;
-}

package/sources/T2/UDN.mjs

@@ -1,26 +0,0 @@
-export function UDN($) {
-  const data = $.extract({
-    title: "h1.article-content__title",
-    date: [
-      {
-        selector: "time.article-content__time",
-        value: (el, key) => {
-          const data = $(el).text().replace(/\//g, "-").slice(0, 10);
-          return data;
-        },
-      },
-    ],
-    author: "a[href^='/news/reporter/']",
-    content: [
-      {
-        selector: "section.article-content__editor  p",
-        value: (el, key) => {
-          const text = $(el).text().trim();
-          return text;
-        },
-      },
-    ],
-  });
-  data["source"] = "聯合報";
-  return data;
-}

package/sources/T3/SETN.mjs

@@ -1,35 +0,0 @@
-export function SETN($) {
-  const data = $.extract({
-    title: [{selector: "title", value: (el) => $(el).text().trim()}],
-
-    date: [
-      {
-        selector: "meta[name=lastmod]",
-        value: (el) => $(el).attr("content").split("T")[0],
-      },
-    ],
-    author: [
-      {
-        selector: "div p:not([style]):first",
-        value: (el) =>
-          $(el)
-            .text()
-            .match(/(?<=記者).{3}/)
-            ? $(el)
-                .text()
-                .match(/(?<=記者).{3}/)[0]
-                .trim()
-            : $(el).text(),
-      },
-    ],
-  });
-  data["source"] = "三立新聞網";
-  data["content"] = [];
-
-  $("div p:not([style])")
-    .slice(1)
-    .each(function () {
-      data["content"].push($(this).text());
-    });
-  return data;
-}

package/sources/T4/CNA.mjs

@@ -1,33 +0,0 @@
-export function CNA($) {
-  const data = $.extract({
-    content: [
-      {
-        selector: "div.paragraph > p",
-        value: (el, key) => {
-          return $(el).text();
-        },
-      },
-    ],
-    author: [
-      {
-        selector: "div.paragraph > p",
-        value: (el, key) => {
-          if ( $(el).text().includes("（中央社")) {
-            let index = $(el).text().indexOf("記者");
-
-            // Extract the 3 characters after "記者"
-            return $(el)
-              .text()
-              .substring(index + 2, index + 5);
-            // return index
-          }
-        },
-      },
-    ],
-  });
-  data["date"] = $('meta[property="article:published_time"]')
-    .attr("content")
-    .slice(0, 10);
-  data["source"] = "中央社";
-  return data;
-}

package/sources/T4/ETDAY.mjs

@@ -1,39 +0,0 @@
-export function ETDAY($) {
-  const data = $.extract({
-    title: "h1.title",
-    date: [
-      {
-        selector: "time.date",
-        value: (el, key) => {
-          const date = new Date($(el).attr("datetime"));
-
-          // Format the date to "YYYY-MM-DD"
-          const formattedDate = date.toISOString().split("T")[0];
-          return formattedDate;
-        },
-      },
-    ],
-    author: [
-      {
-        selector: "div > p:first",
-        value: (el) => {
-          const data = $(el).text()
-
-          return data.replace(/.*記者(.{3}).*/, '$1');
-        },
-      },
-    ],
-    content: [
-      {
-        selector: "div.story > p",
-        value: (el, key) => {
-          // if (key > 1) {
-            return $(el).text().trim()
-          // }
-        },
-      },
-    ],
-  });
-  data["source"] = "ETtoday"
-  return data;
-}