mcp-scraper 0.43.5 → 0.43.6

package/dist/chunk-4FQDZ2T7.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.43.6";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-4FQDZ2T7.js.map

package/dist/chunk-4FQDZ2T7.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.43.6'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{extract-bundle-ITPFWRNI.js → extract-bundle-U3MNYDKC.js}

@@ -1,7 +1,7 @@
 import {
   createSiteExtractBundleArtifactStream,
   downloadAsset
-} from "./chunk-6WLNXYRG.js";
+} from "./chunk-27FMOD6S.js";
 import {
   computeIssues,
   renderImageSection,
@@ -565,4 +565,4 @@ export {
   SITE_EXTRACT_PAGE_CHUNK,
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-ITPFWRNI.js.map
+//# sourceMappingURL=extract-bundle-U3MNYDKC.js.map

package/dist/{server-BTDXZA5U.js → server-ZCX5X3N4.js}

@@ -4,7 +4,7 @@ import {
   harvestPageMedia,
   readOwnedSiteExtractArtifactBuffer,
   renewSiteExtractArtifactDownload
-} from "./chunk-6WLNXYRG.js";
+} from "./chunk-27FMOD6S.js";
 import {
   REQUIRED_LOCATION_STATE_CODES,
   importHostedCensusPlacesCsv,
@@ -86,7 +86,7 @@ import {
   renewDirectoryArtifactDownload,
   resolveDeploymentProfile,
   transcribeMediaUrl
-} from "./chunk-RSGUS5V6.js";
+} from "./chunk-3FKUKMNE.js";
 import {
   auditImageUrls,
   auditImages,
@@ -142,7 +142,7 @@ import {
 } from "./chunk-3ZUBQQPQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-H4RPUFRR.js";
+} from "./chunk-4FQDZ2T7.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -3775,6 +3775,9 @@ async function fetchWithKernel(url) {
   }
 }
 
+// src/lib/http-user-agent.ts
+var MCP_SCRAPER_CRAWLER_USER_AGENT = "MCPScraperBot/1.0 (+https://mcpscraper.dev)";
+
 // src/api/kpo-extractor.ts
 async function extractKpo(opts) {
   const start = Date.now();
@@ -3784,7 +3787,7 @@ async function extractKpo(opts) {
       let target = parsed.href;
       for (let redirects = 0; redirects < 5; redirects++) {
         const res = await fetch(target, {
-          headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
+          headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT },
           signal: AbortSignal.timeout(15e3),
           redirect: "manual"
         });
@@ -4505,7 +4508,6 @@ var SKIP_PATH_PREFIXES = [
 ];
 var SKIP_QUERY_PARAMS = ["replytocom", "p", "preview", "ver", "v"];
 var KERNEL_RETRY_LIMIT = 15;
-var UA = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
 var MAX_DISCOVERED_URL_CHARS = 4096;
 var MAX_DISCOVERY_FRONTIER_BYTES = 4 * 1024 * 1024;
 function boundedDiscoveredUrl(value) {
@@ -4557,7 +4559,7 @@ async function fetchText(url, auditedHost, timeoutMs = 1e4) {
       const checked = await validatePublicHttpUrl(target, { field: "sitemap URL" });
       if (checked.error || !checked.parsed || !sameAuditedHost(checked.parsed, auditedHost)) return null;
       const res = await fetch(checked.parsed.href, {
-        headers: { "User-Agent": UA },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT },
         signal: AbortSignal.timeout(timeoutMs),
         redirect: "manual"
       });
@@ -4675,7 +4677,7 @@ async function fetchPage(url, timeoutMs) {
       const checked = await validatePublicHttpUrl(target, { field: "page URL" });
       if (checked.error || !checked.parsed || !sameAuditedHost(checked.parsed, auditedHost)) return { html: null, status: null };
       const res = await fetch(checked.parsed.href, {
-        headers: { "User-Agent": UA, "Accept": "text/html,application/xhtml+xml" },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT, "Accept": "text/html,application/xhtml+xml" },
         signal: AbortSignal.timeout(timeoutMs),
         redirect: "manual"
       });
@@ -5621,7 +5623,6 @@ function makeSeedSpider(startUrl, urls) {
     sitemapUrls: []
   };
 }
-var UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
 var EXTRACT_CONCURRENCY = 6;
 var MAX_PAGE_MARKDOWN = 4e4;
 var PAGE_DATA_LIMITS = {
@@ -5983,7 +5984,7 @@ async function fetchPagePlain(url) {
   try {
     for (let redirects = 0; redirects <= 5; redirects++) {
       const res = await fetch(target, {
-        headers: { "User-Agent": UA2, "Accept": "text/html,application/xhtml+xml" },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT, "Accept": "text/html,application/xhtml+xml" },
         signal: AbortSignal.timeout(15e3),
         redirect: "manual"
       });
@@ -8666,7 +8667,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-ITPFWRNI.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-U3MNYDKC.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -37263,7 +37264,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
   } = await import("./site-extract-repository-CKCJDP7N.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-ITPFWRNI.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-U3MNYDKC.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -37469,4 +37470,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-BTDXZA5U.js.map
+//# sourceMappingURL=server-ZCX5X3N4.js.map