mcp-scraper 0.76.0 → 0.78.0

package/dist/{chunk-7RQULQF3.js → chunk-B6L73AO4.js}

@@ -1,7 +1,7 @@
 import {
   publicErrorMessage,
   sanitizeVendorName
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {

package/dist/{chunk-WFQ2E4WE.js → chunk-BTQ23KCT.js}

@@ -8,7 +8,7 @@ import {
 import {
   buildPublicErrorEnvelope,
   getDb
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/lib/media-extractor.ts
 import { createWriteStream, mkdirSync, rmSync } from "fs";

package/dist/{chunk-SDX6TRJ6.js → chunk-FODYEDZK.js}

@@ -12,13 +12,13 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "./chunk-BOGRXBH2.js";
+} from "./chunk-2P2ZVRPK.js";
 import {
   renderInstallTerminal
-} from "./chunk-XSYEDC25.js";
+} from "./chunk-IYYRSVRQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-JNFOBQVH.js";
+} from "./chunk-ZUCU2A6V.js";
 
 // src/mcp/stdio-runtime.ts
 import { readFileSync } from "fs";

package/dist/{chunk-PJEEKOUM.js → chunk-FUMG43EN.js}

@@ -1,7 +1,7 @@
 import {
   getDb,
   getUserByEmail
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/api/session.ts
 import { createHmac, timingSafeEqual } from "crypto";

package/dist/{chunk-XSYEDC25.js → chunk-IYYRSVRQ.js}

@@ -1,6 +1,6 @@
 // release-message.json
 var release_message_default = {
-  message: "MCP Scraper now ships a fail-closed 34-tool restricted entrypoint for Thorbit while the normal package and Desktop extension remain full-surface."
+  message: "PAA harvests now distinguish a proven end of the question frontier from an interrupted browser, automatically resume the same durable job, and preserve captured questions, answers, AI Overview citations, and ranked SERP links."
 };
 
 // src/install-terminal.ts

package/dist/{chunk-A66DGFOU.js → chunk-KAGHJQFO.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/api/lead-list-enrichment-repository.ts
 import { createHash, randomUUID } from "crypto";

package/dist/{chunk-CXY5WV45.js → chunk-NGLZ6UJR.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/api/cost-context.ts
 import { AsyncLocalStorage } from "async_hooks";

package/dist/{chunk-WEFPBAAG.js → chunk-OU4FV2RP.js}

@@ -367,6 +367,9 @@ var RequestAbortedError = class extends Error {
     super(message);
   }
 };
+var PaaInteractionInterruptedError = class extends Error {
+  name = "PaaInteractionInterruptedError";
+};
 var LocationMismatchError = class extends Error {
   name = "LocationMismatchError";
   constructor(message = "Google returned results for a different location than requested") {
@@ -2595,22 +2598,40 @@ async function claimPendingJob() {
   });
   return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
 }
+var DURABLE_PAA_LEASE_MS = 45e3;
 async function claimDurablePaaJob(id) {
   const db = getDb();
   const current = await getJob(id);
   if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
+  const recoveryCount = Math.max(0, Number(current.options.paaRecoveryCount ?? 0));
   if (current.status === "running") return void 0;
   if (current.status === "pending") {
+    const leaseToken = randomUUID();
+    const now = (/* @__PURE__ */ new Date()).toISOString();
+    const leaseExpiresAt = new Date(Date.now() + DURABLE_PAA_LEASE_MS).toISOString();
     const updated = await db.execute({
       sql: `UPDATE jobs
-            SET status = 'running', started_at = COALESCE(started_at, datetime('now')), error = NULL
+            SET status = 'running', started_at = datetime('now'), error = NULL,
+                options = json_set(options,
+                  '$.paaLeaseToken', ?,
+                  '$.paaLeaseExpiresAt', ?,
+                  '$.paaLastDispatchAt', ?)
             WHERE id = ? AND status = 'pending'`,
-      args: [id]
+      args: [leaseToken, leaseExpiresAt, now, id]
     });
     if (updated.rowsAffected === 0) return void 0;
-    return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
+    return {
+      job: {
+        ...current,
+        status: "running",
+        started_at: now,
+        options: { ...current.options, paaLeaseToken: leaseToken, paaLeaseExpiresAt: leaseExpiresAt, paaLastDispatchAt: now }
+      },
+      leaseToken,
+      recoveryCount
+    };
   }
-  return current;
+  return { job: current, leaseToken: null, recoveryCount };
 }
 async function checkpointJob(id, result) {
   const updated = await getDb().execute({
@@ -2619,6 +2640,91 @@ async function checkpointJob(id, result) {
   });
   return updated.rowsAffected === 1;
 }
+async function checkpointDurablePaaJob(id, leaseToken, result) {
+  const leaseExpiresAt = new Date(Date.now() + DURABLE_PAA_LEASE_MS).toISOString();
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET result = ?, options = json_set(options, '$.paaLeaseExpiresAt', ?)
+          WHERE id = ? AND status = 'running' AND json_extract(options, '$.paaLeaseToken') = ?`,
+    args: [JSON.stringify(result), leaseExpiresAt, id, leaseToken]
+  });
+  return updated.rowsAffected === 1;
+}
+async function renewDurablePaaLease(id, leaseToken) {
+  const leaseExpiresAt = new Date(Date.now() + DURABLE_PAA_LEASE_MS).toISOString();
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET options = json_set(options, '$.paaLeaseExpiresAt', ?)
+          WHERE id = ? AND status = 'running' AND json_extract(options, '$.paaLeaseToken') = ?`,
+    args: [leaseExpiresAt, id, leaseToken]
+  });
+  return updated.rowsAffected === 1;
+}
+async function completeDurablePaaJob(id, leaseToken, result) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'done', result = ?, error = NULL, public_error_json = NULL,
+            completed_at = datetime('now'),
+            options = json_set(options, '$.paaLeaseExpiresAt', NULL)
+          WHERE id = ? AND status = 'running' AND json_extract(options, '$.paaLeaseToken') = ?`,
+    args: [JSON.stringify(result), id, leaseToken]
+  });
+  return updated.rowsAffected === 1;
+}
+async function failDurablePaaJob(id, leaseToken, error, publicError) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now'),
+            options = json_set(options, '$.paaLeaseExpiresAt', NULL)
+          WHERE id = ? AND status = 'running' AND json_extract(options, '$.paaLeaseToken') = ?`,
+    args: [error, serializePublicErrorEnvelope(publicError), id, leaseToken]
+  });
+  return updated.rowsAffected === 1;
+}
+async function requeueExpiredDurablePaaJob(id, maxRecoveries) {
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'pending', started_at = NULL,
+            result = CASE WHEN json_valid(result) THEN json_set(result,
+              '$.progress.lifecycle.discoveryStatus', 'interrupted',
+              '$.progress.lifecycle.completionReason', 'The previous worker lease expired; the same job is recovering automatically.',
+              '$.progress.lifecycle.automaticRetries', COALESCE(json_extract(options, '$.paaRecoveryCount'), 0) + 1)
+              ELSE result END,
+            options = json_set(options,
+              '$.paaRecoveryCount', COALESCE(json_extract(options, '$.paaRecoveryCount'), 0) + 1,
+              '$.paaLeaseToken', NULL,
+              '$.paaLeaseExpiresAt', NULL,
+              '$.paaRecoveryQueuedAt', ?)
+          WHERE id = ? AND status = 'running'
+            AND (
+              datetime(json_extract(options, '$.paaLeaseExpiresAt')) <= datetime('now')
+              OR (json_extract(options, '$.paaLeaseExpiresAt') IS NULL AND started_at <= datetime('now', '-15 minutes'))
+            )
+            AND COALESCE(json_extract(options, '$.paaRecoveryCount'), 0) < ?`,
+    args: [now, id, Math.max(0, Math.trunc(maxRecoveries))]
+  });
+  return updated.rowsAffected === 1;
+}
+async function completeExpiredDurablePaaJob(id, result) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'done', result = ?, error = NULL, public_error_json = NULL, completed_at = datetime('now')
+          WHERE id = ? AND status = 'running'
+            AND (
+              datetime(json_extract(options, '$.paaLeaseExpiresAt')) <= datetime('now')
+              OR (json_extract(options, '$.paaLeaseExpiresAt') IS NULL AND started_at <= datetime('now', '-15 minutes'))
+            )`,
+    args: [JSON.stringify(result), id]
+  });
+  return updated.rowsAffected === 1;
+}
+async function failExpiredDurablePaaJob(id, error, publicError) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
+          WHERE id = ? AND status = 'running'
+            AND (
+              datetime(json_extract(options, '$.paaLeaseExpiresAt')) <= datetime('now')
+              OR (json_extract(options, '$.paaLeaseExpiresAt') IS NULL AND started_at <= datetime('now', '-15 minutes'))
+            )`,
+    args: [error, serializePublicErrorEnvelope(publicError), id]
+  });
+  return updated.rowsAffected === 1;
+}
 async function listRecentTerminalDurablePaaJobs(limit = 25) {
   const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
   const rows = await getDb().execute({
@@ -2646,6 +2752,27 @@ async function listStaleRunningDurablePaaJobs(limit = 25) {
   });
   return rows.rows.map((row) => deserialize(rowToRawJob(row)));
 }
+async function listRecoverableDurablePaaJobs(limit = 25) {
+  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
+  const rows = await getDb().execute({
+    sql: `SELECT * FROM jobs
+          WHERE json_extract(options, '$.executionOwner') = 'direct'
+            AND json_extract(options, '$.serpOnly') IS NOT 1
+            AND (
+              (status = 'running' AND (
+                datetime(json_extract(options, '$.paaLeaseExpiresAt')) <= datetime('now')
+                OR (json_extract(options, '$.paaLeaseExpiresAt') IS NULL AND started_at <= datetime('now', '-15 minutes'))
+              ))
+              OR
+              (status = 'pending' AND COALESCE(json_extract(options, '$.paaRecoveryCount'), 0) > 0
+                AND datetime(COALESCE(json_extract(options, '$.paaLastDispatchAt'), '1970-01-01')) <= datetime('now', '-30 seconds'))
+            )
+          ORDER BY COALESCE(started_at, created_at)
+          LIMIT ?`,
+    args: [safeLimit]
+  });
+  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
+}
 async function markJobBillingSettled(id, billedMc, settledAt = (/* @__PURE__ */ new Date()).toISOString()) {
   if (!Number.isSafeInteger(billedMc) || billedMc < 0) throw new Error("job billed amount must be a non-negative integer");
   const updated = await getDb().execute({
@@ -3699,6 +3826,7 @@ export {
   ExtractionError,
   MapsSearchIncompletePageError,
   RequestAbortedError,
+  PaaInteractionInterruptedError,
   LocationMismatchError,
   CREDIT_LOTS_EPOCH,
   CORE_SCHEMA_VERSION,
@@ -3777,8 +3905,16 @@ export {
   claimPendingJob,
   claimDurablePaaJob,
   checkpointJob,
+  checkpointDurablePaaJob,
+  renewDurablePaaLease,
+  completeDurablePaaJob,
+  failDurablePaaJob,
+  requeueExpiredDurablePaaJob,
+  completeExpiredDurablePaaJob,
+  failExpiredDurablePaaJob,
   listRecentTerminalDurablePaaJobs,
   listStaleRunningDurablePaaJobs,
+  listRecoverableDurablePaaJobs,
   markJobBillingSettled,
   completeJob,
   completeRunningJob,

package/dist/{chunk-F5TK4KMT.js → chunk-OUBH6XMY.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-RJ7JVYKU.js";
 import {
   getDb
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/api/location-data-repository.ts
 import { createHash, randomUUID } from "crypto";

package/dist/{chunk-77LQV73B.js → chunk-OWF2JJKN.js}

@@ -90,11 +90,23 @@ var MapsSearchOptionsSchema = z.object({
   headless: z.boolean().default(true)
 });
 var RawPAAItemSchema = z.object({
+  questionId: z.string().min(1),
+  googleLinkId: z.string().nullable(),
+  googleEvidenceId: z.string().nullable(),
   question: z.string().min(1),
   answer: z.string().optional(),
   sourceTitle: z.string().optional(),
   sourceSite: z.string().optional(),
-  sourceCite: z.string().optional()
+  sourceCite: z.string().optional(),
+  sources: z.array(z.object({
+    title: z.string().nullable(),
+    site: z.string().nullable(),
+    url: z.string().min(1),
+    rawUrl: z.string().min(1),
+    resolvedUrl: z.string().nullable(),
+    linkType: z.enum(["plain", "google_url_redirect", "google_goto_redirect"]),
+    resolutionStatus: z.enum(["not_needed", "resolved", "unresolved", "rejected"])
+  })).default([])
 });
 var RawMapsOverviewSchema = z.object({
   name: z.string().nullable(),