dms-scrape 0.2.1 → 0.3.0

package/lib/fetchAndScrape.mjs

@@ -1,19 +0,0 @@
-import { justScrape } from "./justScrape.mjs";
-export async function fetchAndScrape(link) {
-    var response;
-    try {
-      // if (
-        // link.includes("digitimes") ||
-      //   link.includes("chinatimes") ||
-      //   link.includes("ctee") ||
-      //   link.includes("buzzorange")
-      // ) {
-      //   return "Must use extension";
-      // } else {
-        response = await fetch(link);
-        return justScrape(link, await response.text());
-      // }
-    } catch (error) {
-      return error;
-    }
-  }

package/lib/justScrape.mjs

@@ -1,33 +0,0 @@
-// import { processHTML } from "./determineSource.mjs";
-import { filterContent } from "./filterContent.mjs";
-import { determineCategory } from "./determineCategory.mjs";
-import { determineSource } from "./determineSource.mjs";
-import * as cheerio from 'cheerio'
-export function justScrape(link, html) {
-  try {
-    const handlerFunction = determineSource(link);
-    var scrapedContent =
-      typeof handlerFunction == "function"
-        ? handlerFunction(cheerio.load(html))
-        : handlerFunction;
-    if (scrapedContent["error"] == undefined) {
-      scrapedContent["content"]
-        ? (scrapedContent["content"] = filterContent(scrapedContent["content"]))
-        : "";
-
-      Array.isArray(scrapedContent["author"])
-        ? (scrapedContent["author"] = scrapedContent["author"].toString())
-        : "";
-      Array.isArray(scrapedContent["date"])
-        ? (scrapedContent["date"] = scrapedContent["date"].toString())
-        : "";
-      scrapedContent["category"] = determineCategory(scrapedContent["title"]);
-      scrapedContent["url"] = link;
-      scrapedContent["content"] = scrapedContent["content"].join("\n\n");
-    }
-
-    return scrapedContent;
-  } catch (error) {
-    return error;
-  }
-}

package/sources/handleUnsupported.mjs

@@ -1,6 +0,0 @@
-export function handleUnsupported(url) {
-    return {
-        error: "unsupported link",
-        url: url
-    }
-}