meodp 0.0.1

package/dist/index.mjs

@@ -0,0 +1,45 @@
+import { g as getBrowser, s as siteUrlMap } from './shared/meodp.faa3c78c.mjs';
+export { L as LocalLog, P as PQueueMap, a as checkLink, c as checkMEODPUrl, f as checkSite, e as checkSiteUrl, d as checkUrlNomoduleAssets, k as createMEODPLogger, m as createWinstonLogger, h as defaultMEODPConfig, i as defineConfig, w as emojiMap, q as formatArgs, t as getFormattedDataFromResponse, x as getMEODPUrlItemInfo, b as getSiteLinks, u as isLink, j as localLog, l as logUrlItemInfo, n as multiBar, v as parseUrlMap, p as progressBarMap, o as registerPageEvents, r as runByConfig } from './shared/meodp.faa3c78c.mjs';
+import 'consola';
+import 'p-queue';
+import 'consola/utils';
+import 'playwright';
+import 'node:path';
+import 'node:process';
+import 'date-fns';
+import 'fs-extra';
+import 'strip-ansi';
+import 'winston';
+import 'winston/lib/winston/transports';
+import 'cli-progress';
+import 'cli-progress/lib/options';
+import 'cli-progress/lib/format-bar';
+import 'cli-progress/lib/format-time';
+
+async function checkSiteMap(urlItem) {
+  const { url } = urlItem;
+  const robotsUrl = new URL("/robots.txt", url).href;
+  const browser = await getBrowser();
+  const page = await browser.newPage();
+  await page.goto(robotsUrl);
+  const text = await page.textContent("pre");
+  const sitemap = text?.match(/Sitemap: (.*)/)?.[1];
+  let sitemapUrl = new URL(sitemap || "/sitemap.xml", url).href;
+  if (sitemap) {
+    sitemapUrl = sitemap.startsWith("http") ? sitemap : new URL(sitemap, url).href;
+  }
+  await page.goto(sitemapUrl);
+  const links = await page.$$eval("loc", (elements) => {
+    return elements.map((element) => element.textContent);
+  });
+  for (const link of links) {
+    if (link && !siteUrlMap.has(link)) {
+      siteUrlMap.set(link, {
+        statusCode: 0,
+        checkStatus: "pending"
+      });
+    }
+  }
+}
+
+export { checkSiteMap, getBrowser, siteUrlMap };