mcp-scraper 0.3.47 → 0.4.1

package/dist/{server-H6GAAEQ6.js → server-U7V6DUSY.js}

@@ -1,24 +1,47 @@
+import {
+  classifyHarvestProblem,
+  createHarvestAttemptRecorder,
+  harvestProblemResponse,
+  serializeHarvestProblem
+} from "./chunk-3UH3BEIZ.js";
+import {
+  BrowserDriver,
+  MapsSelectors,
+  buildYouTubeChannelVideosUrl,
+  currentCostContext,
+  deleteKernelProxyId,
+  harvest,
+  migrateCostTelemetry,
+  recordKernelSession,
+  recordVendorUsage,
+  resolveKernelProxyId,
+  runWithCostContext
+} from "./chunk-HBC4C75S.js";
 import {
   HttpMcpToolExecutor,
-  auditImages,
-  buildLinkGraph,
-  buildLinkReport,
+  MemoryMcpToolExecutor,
   buildPaaExtractorMcpServer,
-  computeIssues,
   configureReportSaving,
   exportFanout,
-  getBlobStore,
   harvestTimeoutBudget,
   hashOwnerId,
   outputBaseDir,
   registerBrowserAgentMcpTools,
+  registerMemoryMcpTools,
   registerSerpIntelligenceCaptureTools,
-  renderImageSection,
-  renderIssueReport,
-  renderLinkReport,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-A6BAV444.js";
+} from "./chunk-QUOZ3LZG.js";
+import {
+  auditImages,
+  getBlobStore
+} from "./chunk-GWRIO6JT.js";
+import {
+  browserServiceApiKey,
+  browserServiceProfileName,
+  browserServiceProfileSaveChanges,
+  browserServiceProxyId
+} from "./chunk-HPV4VOQX.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -31,7 +54,29 @@ import {
   workflowStepCount,
   workflowSupportsSteps
 } from "./chunk-NNEIXK5L.js";
-import "./chunk-NTCAVBHU.js";
+import "./chunk-K443GQY5.js";
+import {
+  DEFAULT_MAPS_PROXY_MODE,
+  DEFAULT_PROXY_MODE,
+  MapsPlaceOptionsSchema,
+  MapsSearchOptionsSchema,
+  RawMapsAboutAttributeSchema,
+  RawMapsHoursRowSchema,
+  RawMapsOverviewSchema,
+  RawMapsReviewStatsSchema
+} from "./chunk-BP27CZ5Q.js";
+import "./chunk-2ULBZRQQ.js";
+import {
+  completeExtractJob,
+  countSuccessfulPages,
+  createExtractJob,
+  failExtractJob,
+  getExtractJob,
+  getExtractedPages,
+  saveExtractPages,
+  setExtractJobTotal,
+  settleExtractJob
+} from "./chunk-JOEZJL4I.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -48,46 +93,14 @@ import {
   SUBSCRIPTION_TIERS,
   SUBSCRIPTION_TIER_BY_KEY,
   browserActiveCostMc,
-  classifyHarvestProblem,
   concurrencySlotBillingInfo,
-  createHarvestAttemptRecorder,
-  harvestProblemResponse,
-  insufficientBalanceResponse,
-  serializeHarvestProblem
-} from "./chunk-7TFJJAPE.js";
-import {
-  BrowserDriver,
-  MapsSelectors,
-  buildYouTubeChannelVideosUrl,
-  currentCostContext,
-  deleteKernelProxyId,
-  harvest,
-  migrateCostTelemetry,
-  recordKernelSession,
-  recordVendorUsage,
-  resolveKernelProxyId,
-  runWithCostContext
-} from "./chunk-RXNHY3TF.js";
+  insufficientBalanceResponse
+} from "./chunk-6TDUY7CA.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
-  browserServiceApiKey,
-  browserServiceProfileName,
-  browserServiceProfileSaveChanges,
-  browserServiceProxyId,
   sanitizeVendorName
-} from "./chunk-3PRO376E.js";
-import "./chunk-K443GQY5.js";
-import {
-  DEFAULT_MAPS_PROXY_MODE,
-  DEFAULT_PROXY_MODE,
-  MapsPlaceOptionsSchema,
-  MapsSearchOptionsSchema,
-  RawMapsAboutAttributeSchema,
-  RawMapsHoursRowSchema,
-  RawMapsOverviewSchema,
-  RawMapsReviewStatsSchema
-} from "./chunk-BP27CZ5Q.js";
+} from "./chunk-M2S27J6Z.js";
 import {
   SiteAuditJobRowSchema,
   advanceWorkflowRunStep,
@@ -7422,135 +7435,8 @@ var siteAuditFn = inngest.createFunction(
   }
 );
 
-// src/api/site-extract-repository.ts
-function rowToJob(r) {
-  return {
-    id: String(r.id),
-    userId: r.user_id != null ? Number(r.user_id) : null,
-    status: r.status != null ? String(r.status) : "pending",
-    startUrl: String(r.start_url ?? ""),
-    options: r.options ? JSON.parse(String(r.options)) : {},
-    totalUrls: Number(r.total_urls ?? 0),
-    doneUrls: Number(r.done_urls ?? 0),
-    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
-    error: r.error != null ? String(r.error) : null,
-    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
-    createdAt: String(r.created_at ?? ""),
-    updatedAt: String(r.updated_at ?? "")
-  };
-}
-async function createExtractJob(jobId, userId, startUrl, options) {
-  const db = getDb();
-  await db.execute({
-    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
-    args: [jobId, userId, startUrl, JSON.stringify(options)]
-  });
-}
-async function getExtractJob(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
-  return res.rows[0] ? rowToJob(res.rows[0]) : null;
-}
-async function setExtractJobTotal(jobId, totalUrls) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET status = 'running', total_urls = ?, updated_at = datetime('now') WHERE id = ?`,
-    args: [totalUrls, jobId]
-  });
-}
-async function saveExtractPages(jobId, pages) {
-  if (pages.length === 0) return;
-  const db = getDb();
-  await db.batch(
-    pages.map((p) => ({
-      sql: `INSERT OR REPLACE INTO site_extract_pages (job_id, url, page) VALUES (?, ?, ?)`,
-      args: [jobId, p.url, JSON.stringify({ ...p, bodyMarkdown: "", schema: [] })]
-    }))
-  );
-  await db.execute({
-    sql: `UPDATE site_extract_jobs
-            SET done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?), updated_at = datetime('now')
-          WHERE id = ?`,
-    args: [jobId, jobId]
-  });
-}
-async function getExtractedPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
-  return res.rows.map((r) => JSON.parse(String(r.page)));
-}
-async function countSuccessfulPages(jobId) {
-  const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT COUNT(*) AS n FROM site_extract_pages
-            WHERE job_id = ? AND json_extract(page, '$.status') = 200 AND json_extract(page, '$.wordCount') > 0`,
-    args: [jobId]
-  });
-  return Number(res.rows[0]?.n ?? 0);
-}
-async function completeExtractJob(jobId, artifacts) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET status = 'complete', artifacts = ?, updated_at = datetime('now') WHERE id = ?`,
-    args: [JSON.stringify(artifacts ?? []), jobId]
-  });
-}
-async function failExtractJob(jobId, error) {
-  const db = getDb();
-  await db.execute({
-    sql: `UPDATE site_extract_jobs SET status = 'failed', error = ?, updated_at = datetime('now') WHERE id = ?`,
-    args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
-  });
-}
-async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
-  const db = getDb();
-  const job = await getExtractJob(jobId);
-  if (!job || job.billedMc != null) return;
-  if (refundMc <= 0) {
-    await db.execute({ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] });
-    return;
-  }
-  await db.batch([
-    { sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [refundMc, userId] },
-    { sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)", args: [userId, refundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference] },
-    { sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] }
-  ]);
-}
-
 // src/inngest/functions/site-extract.ts
 var BATCH = 30;
-function buildArtifacts(job, pages, branding, imageAudit) {
-  const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
-  const issues = computeIssues(pages, metrics);
-  const linkReport = buildLinkReport(edges, [...metrics.values()], job.startUrl);
-  let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
-  if (imageAudit) reportMd += `
-
-${renderImageSection(imageAudit)}`;
-  const pageRows = pages.map((p) => {
-    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
-    const m = metrics.get(p.url);
-    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
-  });
-  const jsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
-  const files = [
-    { name: "report.md", body: reportMd, type: "text/markdown" },
-    { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
-    { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
-    { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
-    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" },
-    { name: "link-report.md", body: renderLinkReport(linkReport), type: "text/markdown" },
-    { name: "links-summary.json", body: JSON.stringify(linkReport.summary, null, 2), type: "application/json" },
-    { name: "external-domains.json", body: JSON.stringify(linkReport.externalDomains, null, 2), type: "application/json" }
-  ];
-  if (imageAudit) {
-    files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
-    files.push({ name: "images-summary.json", body: JSON.stringify(imageAudit.summary, null, 2), type: "application/json" });
-  }
-  if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
-  return files;
-}
 var siteExtractFn = inngest.createFunction(
   {
     id: "site-extract",
@@ -7565,7 +7451,9 @@ var siteExtractFn = inngest.createFunction(
         await settleExtractJob(jobId, current.userId, heldMc, 0, "crawl failed refund").catch(() => {
         });
       }
-      await failExtractJob(jobId, String(event?.data?.error?.message ?? "crawl failed")).catch(() => {
+      const rawErr = String(event?.data?.error?.message ?? "crawl failed");
+      const friendly = /FUNCTION_INVOCATION_FAILED|out of memory|heap/i.test(rawErr) ? "Result assembly failed on our side \u2014 your crawl data is safe and the job can be re-finalized without re-crawling. Contact support if this persists." : rawErr;
+      await failExtractJob(jobId, friendly).catch(() => {
       });
     }
   },
@@ -7641,11 +7529,8 @@ var siteExtractFn = inngest.createFunction(
       return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
     }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const pages = await getExtractedPages(jobId);
-      const store = getBlobStore();
-      const files = buildArtifacts(job, pages, branding, imageAudit);
-      const stored = [];
-      for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
+      const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
+      const stored = await assembleExtractArtifacts(job, { branding, imageAudit });
       await completeExtractJob(jobId, stored);
       return stored;
     });
@@ -11961,8 +11846,13 @@ function verifySession(token) {
 }
 
 // src/api/memory.ts
-var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://mcp-memory-omega.vercel.app").replace(/\/$/, "");
+var MEMORY_BASE_URL = () => (process.env.MCP_MEMORY_BASE_URL ?? "https://memory.mcpscraper.dev").replace(/\/$/, "");
 var ADMIN_KEY = () => process.env.MCP_MEMORY_ADMIN_KEY?.trim() ?? "";
+function parseMemoryRpcBody(text) {
+  const dataLine = text.split("\n").find((line) => line.startsWith("data:"));
+  const payload = dataLine ? dataLine.slice(5).trim() : text;
+  return JSON.parse(payload);
+}
 function isMemoryOperator(email) {
   if (!email) return false;
   const ops = (process.env.MEMORY_OPERATOR_IDENTITIES ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean);
@@ -11990,15 +11880,34 @@ function decryptMemoryKey(stored) {
 }
 async function memoryCall(toolName, args, userMemoryKey) {
   try {
-    const res = await fetch(`${MEMORY_BASE_URL()}/api/mcp/memoryServer/tools/${toolName}/execute`, {
+    const res = await fetch(`${MEMORY_BASE_URL()}/mcp`, {
       method: "POST",
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify({ data: { ...args, apiKey: userMemoryKey } })
+      headers: {
+        "content-type": "application/json",
+        accept: "application/json, text/event-stream",
+        authorization: `Bearer ${userMemoryKey}`
+      },
+      body: JSON.stringify({
+        jsonrpc: "2.0",
+        id: 1,
+        method: "tools/call",
+        params: { name: toolName, arguments: args }
+      })
     });
-    const body = await res.json().catch(() => null);
-    if (!res.ok) return { ok: false, error: body?.error ?? `memory ${toolName} failed (${res.status})` };
-    if (body?.result) return body.result;
-    return { ok: false, error: body?.error ?? `memory ${toolName} returned no result` };
+    const text = await res.text();
+    if (!res.ok) return { ok: false, error: `memory ${toolName} failed (${res.status})` };
+    const body = parseMemoryRpcBody(text);
+    if (body.error) return { ok: false, error: body.error.message ?? `memory ${toolName} failed` };
+    if (body.result?.structuredContent) return body.result.structuredContent;
+    const textContent = body.result?.content?.find((c) => c.type === "text")?.text;
+    if (textContent) {
+      try {
+        return JSON.parse(textContent);
+      } catch {
+        return { ok: false, error: textContent };
+      }
+    }
+    return { ok: false, error: `memory ${toolName} returned no result` };
   } catch (err) {
     return { ok: false, error: err?.message ?? "memory call failed" };
   }
@@ -15096,6 +15005,7 @@ mcpApp.all("/", async (c) => {
     const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
     registerSerpIntelligenceCaptureTools(server, executor);
     registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
+    registerMemoryMcpTools(server, new MemoryMcpToolExecutor(baseUrl, callerKey));
     await server.connect(transport);
     return transport.handleRequest(c.req.raw);
   } catch {
@@ -19484,6 +19394,15 @@ app.get("/memory/vaults", auth2, async (c) => {
     return c.json({ ok: false, error: err instanceof Error ? err.message : "could not load vaults" }, 502);
   }
 });
+app.post("/memory/mcp-call", auth2, async (c) => {
+  const user = c.get("user");
+  const body = await c.req.json().catch(() => ({}));
+  if (!body.toolName) return c.json({ ok: false, error: "toolName is required" }, 400);
+  const { key, error } = await getOrCreateUserMemoryKey(user);
+  if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 502);
+  const result = await memoryCall(body.toolName, body.args ?? {}, key);
+  return c.json(result);
+});
 app.get("/memory/approved-senders", auth2, async (c) => {
   const { key, error } = await getOrCreateUserMemoryKey(c.get("user"));
   if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 502);
@@ -20548,7 +20467,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-WY7C45MP.js");
+  const { drainQueue } = await import("./worker-RVZXIALX.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -20560,6 +20479,35 @@ app.get("/cron/tick", async (c) => {
   ]);
   return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, expired: expiredResult, blobCleanup, workflowDispatch: workflowDispatchResult });
 });
+app.post("/api/internal/extract-refinalize/:id", async (c) => {
+  const secret2 = c.req.header("authorization");
+  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
+    return c.json({ error: "Unauthorized" }, 401);
+  }
+  const jobId = c.req.param("id");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-2MQBAOX5.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
+  const job = await getExtractJob2(jobId);
+  if (!job) return c.json({ error: "job not found" }, 404);
+  const stored = await assembleExtractArtifacts(job, {});
+  await completeExtractJob2(jobId, stored);
+  let settlement = "already_settled_or_refunded";
+  if (job.billedMc == null && job.userId != null) {
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-2MQBAOX5.js");
+    const heldMc = Number(job.options.heldMc ?? 0);
+    const successful = await countSuccessfulPages2(jobId);
+    const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
+    let host = "site";
+    try {
+      host = new URL(job.startUrl).hostname;
+    } catch {
+    }
+    await settleExtractJob2(jobId, job.userId, heldMc - usedMc, usedMc, host);
+    settlement = `settled: billed ${usedMc} mc, refunded ${heldMc - usedMc} mc`;
+  }
+  const bundle = stored.find((a) => a.key.endsWith("bundle.zip"));
+  return c.json({ ok: true, jobId, artifacts: stored.length, settlement, bundleUrl: bundle?.url ?? null, bundleBytes: bundle?.bytes ?? null });
+});
 app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
 app.route("/api/internal/site-architecture-auditor", siteAuditApp);
 app.route("/api/internal/memory", internalMemoryApp);
@@ -20701,4 +20649,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-H6GAAEQ6.js.map
+//# sourceMappingURL=server-U7V6DUSY.js.map