mcp-scraper 0.4.5 → 0.4.6

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-NJK5BTUK.js";
+} from "./chunk-5GAD2AN2.js";
 import "./chunk-HPV4VOQX.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-XGIPATLV.js";
 import "./chunk-M2S27J6Z.js";
-import "./chunk-FUVQR6GK.js";
+import "./chunk-GL4BW4CP.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{server-XCDFHHLE.js → server-R3KVQNOT.js}

@@ -3,7 +3,7 @@ import {
   createHarvestAttemptRecorder,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-3UH3BEIZ.js";
+} from "./chunk-HE2LQPJ2.js";
 import {
   BrowserDriver,
   MapsSelectors,
@@ -16,7 +16,7 @@ import {
   recordVendorUsage,
   resolveKernelProxyId,
   runWithCostContext
-} from "./chunk-NJK5BTUK.js";
+} from "./chunk-5GAD2AN2.js";
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
@@ -31,7 +31,7 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-WI4A3KFQ.js";
+} from "./chunk-IFXACD4K.js";
 import {
   auditImages,
   getBlobStore
@@ -65,7 +65,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-UGQC2FOX.js";
+import "./chunk-5BB6Y3BB.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -76,7 +76,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-4ZZWFI6L.js";
+} from "./chunk-NYAWN7CZ.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -95,7 +95,7 @@ import {
   browserActiveCostMc,
   concurrencySlotBillingInfo,
   insufficientBalanceResponse
-} from "./chunk-5LO6EHEH.js";
+} from "./chunk-IVQYNY45.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -131,6 +131,7 @@ import {
   getLedger,
   getMemoryPlan,
   getMemoryProvisioned,
+  getPageSnapshot,
   getRefresh,
   getUserByApiKey,
   getUserByEmail,
@@ -175,8 +176,9 @@ import {
   setStripeCustomerId,
   setSubscriptionTier,
   setUserMemoryCreds,
+  upsertPageSnapshot,
   verifyPassword
-} from "./chunk-FUVQR6GK.js";
+} from "./chunk-GL4BW4CP.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -7529,7 +7531,7 @@ var siteExtractFn = inngest.createFunction(
       return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
     }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
       const stored = await assembleExtractArtifacts(job, { branding, imageAudit });
       await completeExtractJob(jobId, stored);
       return stored;
@@ -8935,6 +8937,11 @@ var ExtractUrlBodySchema = z11.object({
   depositToVault: z11.boolean().optional(),
   vaultName: z11.string().trim().min(1).max(120).optional()
 });
+var DiffPageBodySchema = z11.object({
+  url: z11.string().min(1, "url is required"),
+  allowLocal: z11.boolean().optional(),
+  resetBaseline: z11.boolean().optional()
+});
 var MapUrlsBodySchema = z11.object({
   url: z11.string().min(1, "url is required"),
   maxUrls: z11.number().int().min(1).max(2e3).optional(),
@@ -19769,6 +19776,71 @@ import { createMiddleware as createMiddleware3 } from "hono/factory";
 import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "hono/cookie";
 import Stripe2 from "stripe";
 
+// src/api/page-diff.ts
+import { createHash as createHash4 } from "crypto";
+import { diffLines } from "diff";
+var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
+var MAX_DIFF_HUNKS = 200;
+var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
+function sha256Hex(value) {
+  return createHash4("sha256").update(value).digest("hex");
+}
+function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
+  if (value.length <= maxChars) return { value, truncated: false };
+  return { value: value.slice(0, maxChars), truncated: true };
+}
+function changeLines(change) {
+  const lines = change.value.split("\n");
+  if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop();
+  return lines;
+}
+function diffPageContent(oldContent, newContent) {
+  const changes = diffLines(oldContent, newContent);
+  const rawHunks = [];
+  let linesAdded = 0;
+  let linesRemoved = 0;
+  let totalChangedLineCount = 0;
+  for (const change of changes) {
+    if (!change.added && !change.removed) continue;
+    const lines = changeLines(change);
+    totalChangedLineCount += lines.length;
+    if (change.added) linesAdded += lines.length;
+    if (change.removed) linesRemoved += lines.length;
+    rawHunks.push({ type: change.added ? "added" : "removed", lines });
+  }
+  let hunksTruncated = false;
+  const hunksWithinCount = rawHunks.length > MAX_DIFF_HUNKS ? rawHunks.slice(0, MAX_DIFF_HUNKS) : rawHunks;
+  if (rawHunks.length > MAX_DIFF_HUNKS) hunksTruncated = true;
+  const hunks = [];
+  let emittedLines = 0;
+  for (const hunk of hunksWithinCount) {
+    if (emittedLines >= MAX_DIFF_LINES_PER_RESPONSE) {
+      hunksTruncated = true;
+      break;
+    }
+    const remaining = MAX_DIFF_LINES_PER_RESPONSE - emittedLines;
+    if (hunk.lines.length > remaining) {
+      hunks.push({ type: hunk.type, lines: hunk.lines.slice(0, remaining) });
+      emittedLines += remaining;
+      hunksTruncated = true;
+      break;
+    }
+    hunks.push(hunk);
+    emittedLines += hunk.lines.length;
+  }
+  const denominator = Math.max(oldContent.split("\n").length, newContent.split("\n").length, 1);
+  const percentChanged = totalChangedLineCount > 0 ? Math.min(100, Math.round(totalChangedLineCount / denominator * 1e3) / 10) : 0;
+  return {
+    hunks,
+    linesAdded,
+    linesRemoved,
+    percentChanged,
+    hunksTruncated,
+    hunksTruncatedReason: hunksTruncated ? `Showing ${emittedLines} of ${totalChangedLineCount} changed lines (capped at ${MAX_DIFF_HUNKS} hunks / ${MAX_DIFF_LINES_PER_RESPONSE} lines). Use contentHash/previousContentHash to confirm the full scope of change.` : null,
+    totalChangedLineCount
+  };
+}
+
 // src/api/scrape-vault-sink.ts
 var MAX_DEPOSIT_CHARS = 5e6;
 var SCRAPE_FALLBACK_PREFIX = "scrape-fallback/";
@@ -21098,6 +21170,91 @@ app.post("/extract-url", auth2, async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+app.post("/diff-page", auth2, async (c) => {
+  const raw = await c.req.json().catch(() => ({}));
+  const bodyResult = DiffPageBodySchema.safeParse(raw);
+  if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
+  const { url, allowLocal, resetBaseline } = bodyResult.data;
+  if (!allowLocal) {
+    const checked = await validatePublicHttpUrl(url, { field: "URL" });
+    if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
+  } else {
+    try {
+      new URL(url);
+    } catch {
+      return c.json({ error: "Invalid URL" }, 400);
+    }
+  }
+  const canonicalUrl = (() => {
+    try {
+      return new URL(url).href;
+    } catch {
+      return url;
+    }
+  })();
+  const user = c.get("user");
+  const gate = await acquireConcurrencyGate(user, "diff_page", {
+    reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
+    metadata: { url: canonicalUrl }
+  });
+  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  let debited = false;
+  try {
+    const { ok: dpOk, balance_mc: dpBal } = await debitMc(user.id, MC_COSTS.diff_page, LedgerOperation.DIFF_PAGE, new URL(canonicalUrl).hostname);
+    if (!dpOk) return c.json(insufficientBalanceResponse(dpBal, MC_COSTS.diff_page), 402);
+    debited = true;
+    const existingSnapshot = await getPageSnapshot(user.id, canonicalUrl);
+    const previous = resetBaseline ? null : existingSnapshot;
+    const kernelApiKey = browserServiceApiKey();
+    const result = await extractKpo({ url: canonicalUrl, kernelApiKey });
+    const currentContent = result.bodyMarkdown ?? "";
+    const currentHash = sha256Hex(currentContent);
+    const { value: storedContent, truncated: contentTruncated } = truncateForStorage(currentContent);
+    const currentCheckedAt = (/* @__PURE__ */ new Date()).toISOString();
+    await upsertPageSnapshot(user.id, canonicalUrl, {
+      contentHash: currentHash,
+      content: storedContent,
+      title: result.title ?? null,
+      contentBytes: Buffer.byteLength(currentContent, "utf8"),
+      truncated: contentTruncated
+    });
+    let status;
+    let diff = { hunks: [], linesAdded: 0, linesRemoved: 0, percentChanged: 0, hunksTruncated: false, hunksTruncatedReason: null, totalChangedLineCount: 0 };
+    if (!previous) {
+      status = "baseline";
+      diff.percentChanged = null;
+    } else if (previous.contentHash === currentHash) {
+      status = "unchanged";
+    } else {
+      status = "changed";
+      diff = diffPageContent(previous.content, storedContent);
+    }
+    await logRequestEvent({ userId: user.id, source: "diff_page", status: "done", query: canonicalUrl, result: { status } });
+    return c.json({
+      url: canonicalUrl,
+      title: result.title ?? null,
+      status,
+      isReset: !!resetBaseline && !!existingSnapshot,
+      previousCheckedAt: previous?.checkedAt ?? null,
+      currentCheckedAt,
+      contentHash: currentHash,
+      previousContentHash: previous?.contentHash ?? null,
+      summary: { linesAdded: diff.linesAdded, linesRemoved: diff.linesRemoved, percentChanged: diff.percentChanged },
+      hunks: diff.hunks,
+      contentTruncated,
+      hunksTruncated: diff.hunksTruncated,
+      hunksTruncatedReason: diff.hunksTruncatedReason,
+      totalChangedLineCount: diff.totalChangedLineCount
+    });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (debited) await creditMc(user.id, MC_COSTS.diff_page, LedgerOperation.DIFF_PAGE, "failed call");
+    await logRequestEvent({ userId: user.id, source: "diff_page", status: "failed", query: canonicalUrl, error: msg });
+    return c.json({ error: msg }, 500);
+  } finally {
+    await releaseConcurrencyGate(gate.lockId);
+  }
+});
 app.post("/map-urls", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = MapUrlsBodySchema.safeParse(raw);
@@ -21524,7 +21681,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-YHKUJR5P.js");
+  const { drainQueue } = await import("./worker-VASAJ7FZ.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21542,15 +21699,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-THBVEXMP.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-LI2S3S6E.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-THBVEXMP.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-LI2S3S6E.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -21714,4 +21871,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-XCDFHHLE.js.map
+//# sourceMappingURL=server-R3KVQNOT.js.map