mcp-scraper 0.77.0 → 0.78.0

package/dist/{server-EKRNKKLU.js → server-6B2OG3NN.js}

@@ -10,7 +10,7 @@ import {
   queuedCommonsDocumentCount,
   semanticCommonsEntityScores,
   typeFromMime
-} from "./chunk-WFQ2E4WE.js";
+} from "./chunk-BTQ23KCT.js";
 import {
   SITE_EXTRACT_ARTIFACT_PREFIX,
   abandonExtractSettlement,
@@ -41,13 +41,13 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-D2O7V2CT.js";
+} from "./chunk-VMWSXKUQ.js";
 import {
   REQUIRED_LOCATION_STATE_CODES,
   importHostedCensusPlacesCsv,
   importHostedZipGroupsCsv,
   listActiveHostedLocationDatasets
-} from "./chunk-F5TK4KMT.js";
+} from "./chunk-OUBH6XMY.js";
 import {
   cancelPendingLeadListEnrichmentRows,
   claimLeadListEnrichmentOutbox,
@@ -74,7 +74,7 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-A66DGFOU.js";
+} from "./chunk-KAGHJQFO.js";
 import {
   ConnectedUsageBillingError,
   ConnectedUsagePreflightInputSchema,
@@ -101,7 +101,7 @@ import {
   probeNangoConnectionDirect,
   recordNangoConnectionCost,
   settleConnectedUsage
-} from "./chunk-E6Y3VCOJ.js";
+} from "./chunk-3PUMNEOC.js";
 import {
   ANALYTICS_CONTENT_SORTS,
   AnalyticsIdentityProfileError,
@@ -185,7 +185,7 @@ import {
   upsertAnalyticsAdSpend,
   upsertAnalyticsHostGroup,
   verifyAnalyticsWebhookSignature
-} from "./chunk-IBQNOZ7E.js";
+} from "./chunk-74CCAY6T.js";
 import {
   claimServiceConnectionAction,
   ensureServiceConnectionsSchema,
@@ -198,7 +198,7 @@ import {
   setServiceConnectionUserLabel,
   signSession,
   verifySession
-} from "./chunk-PJEEKOUM.js";
+} from "./chunk-FUMG43EN.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
@@ -207,7 +207,7 @@ import {
   harvestProblemResponse,
   publicErrorBoundary,
   serializeHarvestProblem
-} from "./chunk-KIPIBPUB.js";
+} from "./chunk-47ZCUBM5.js";
 import "./chunk-OZJMVCDK.js";
 import {
   BACKUP_LAST_RESORT_ATTEMPTS,
@@ -216,18 +216,20 @@ import {
   backupProxyAvailable,
   brightDataSerpEnabled,
   buildYouTubeChannelVideosUrl,
+  classifyGoogleOutboundLink,
   cleanupBackupProxyId,
   createBackupProxyIdSafe,
   deleteKernelProxyId,
   detectPageUserAgent,
   detectRuntimeUserAgent,
   harvest,
+  questionIdFor,
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
   serpArrivalUrl,
   validateProviderSessionId,
   withRuntimeUserAgent
-} from "./chunk-F6MUGMRN.js";
+} from "./chunk-UFJJGXVO.js";
 import {
   AnalyticsConfirmedPersonProjectionSchema,
   AnalyticsCrmFieldSchema,
@@ -314,7 +316,7 @@ import {
   resolveDeploymentProfile,
   resolveLocalSourcebookSchemaType,
   transcribeMediaUrl
-} from "./chunk-DTJ56S5B.js";
+} from "./chunk-2P2ZVRPK.js";
 import {
   auditImageUrls,
   auditImages,
@@ -338,7 +340,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   sanitizeOutboundDiagnostics
-} from "./chunk-7RQULQF3.js";
+} from "./chunk-B6L73AO4.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PACK_INVOICE_DESCRIPTION,
@@ -397,7 +399,7 @@ import {
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-CXY5WV45.js";
+} from "./chunk-NGLZ6UJR.js";
 import {
   listWorkflowDefinitions,
   runWorkflow,
@@ -407,7 +409,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-KBTUPRTF.js";
+} from "./chunk-VEQPA65D.js";
 import {
   csvRecords,
   parseCsv,
@@ -422,7 +424,7 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-77LQV73B.js";
+} from "./chunk-OWF2JJKN.js";
 import {
   extractContactEvidence
 } from "./chunk-VXLU74YZ.js";
@@ -433,7 +435,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-7HF3T7P2.js";
+} from "./chunk-ZUCU2A6V.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -444,11 +446,12 @@ import {
   buildPublicErrorEnvelope,
   cancelJob,
   checkRateLimit,
-  checkpointJob,
+  checkpointDurablePaaJob,
   claimDurablePaaJob,
   claimStripeEvent,
+  completeDurablePaaJob,
+  completeExpiredDurablePaaJob,
   completeJob,
-  completeRunningJob,
   completeStripeEvent,
   completeWorkflowRunRecord,
   consumeCode,
@@ -468,8 +471,9 @@ import {
   debitMcIdempotent,
   deleteWorkflowSchedule,
   expireOldLots,
+  failDurablePaaJob,
+  failExpiredDurablePaaJob,
   failJob,
-  failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
   getBillingLedgerSummary,
@@ -503,8 +507,8 @@ import {
   listInboxMessages,
   listJobs,
   listRecentTerminalDurablePaaJobs,
+  listRecoverableDurablePaaJobs,
   listRequestEvents,
-  listStaleRunningDurablePaaJobs,
   listUsers,
   listUsersDueForConnectedAccountBillingReconciliation,
   listWorkflowArtifacts,
@@ -521,7 +525,9 @@ import {
   reconcileBalanceMc,
   recordWorkflowWebhookDelivery,
   registerClient,
+  renewDurablePaaLease,
   replaceWorkflowArtifacts,
+  requeueExpiredDurablePaaJob,
   resolveUserByMemorySubscription,
   revokeApiKey,
   rotateApiKey,
@@ -540,7 +546,7 @@ import {
   updateHarvestAttemptProviderTelemetry,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-WEFPBAAG.js";
+} from "./chunk-OU4FV2RP.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -9641,7 +9647,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-OO74L67T.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-P2DTUVY3.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10601,7 +10607,7 @@ async function resolveUsZipsPath(requestedPath) {
 async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
   const mode = resolvedLocationDataMode({ zipSource: sourceMode });
   if (mode === "hosted") {
-    const { getHostedZipGroups } = await import("./location-data-repository-H4B2BWGY.js");
+    const { getHostedZipGroups } = await import("./location-data-repository-AHKDXTEQ.js");
     const hosted = await getHostedZipGroups(stateAbbr);
     if (!hosted.dataset) {
       throw new DirectoryLocationError(
@@ -10664,7 +10670,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
   const sourceUrl = censusStateUrl(state.fips);
   const warnings = [];
   if (resolvedLocationDataMode(config) === "hosted") {
-    const { queryHostedLocationMarkets } = await import("./location-data-repository-H4B2BWGY.js");
+    const { queryHostedLocationMarkets } = await import("./location-data-repository-AHKDXTEQ.js");
     const hosted = await queryHostedLocationMarkets({
       state: state.abbr,
       minPopulation: options.minPopulation,
@@ -12151,7 +12157,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
 async function settlePersistedLeadListEnrichment(jobId2) {
   const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
   const job = jobs.find((candidate) => candidate.id === jobId2);
-  const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-WWEMJ7EN.js");
+  const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-5LEWW4X7.js");
   const current = job ?? await getLeadListEnrichmentJob2(jobId2);
   if (!current) return null;
   const rows = await listLeadListEnrichmentRows(jobId2);
@@ -12207,7 +12213,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
         const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
         const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
         if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
-          const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-WWEMJ7EN.js");
+          const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-5LEWW4X7.js");
           await failLeadListEnrichmentJob2({
             jobId: job.id,
             errorCode: "lead_list_all_rows_failed",
@@ -25939,6 +25945,7 @@ var CLEANUP_RESERVE_MS = 4e4;
 var DIRECT_PAA_MAX_ATTEMPTS = 3;
 var CHECKPOINT_INTERVAL_MS = 1e4;
 var CHECKPOINT_IMPROVEMENT_THRESHOLD = 5;
+var LEASE_HEARTBEAT_MS = 1e4;
 function normalizePaaQuestion(value) {
   if (typeof value !== "string") return "";
   return value.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
@@ -25946,45 +25953,87 @@ function normalizePaaQuestion(value) {
 function clean2(value) {
   return typeof value === "string" ? value.trim() : "";
 }
+function normalizeSource(value) {
+  if (!value || typeof value !== "object") return null;
+  const raw = value;
+  const classified = classifyGoogleOutboundLink(clean2(raw.rawUrl) || clean2(raw.url), "https://www.google.com/");
+  if (!classified) return null;
+  return {
+    ...classified,
+    url: clean2(raw.url) || classified.url,
+    rawUrl: clean2(raw.rawUrl) || classified.rawUrl,
+    resolvedUrl: clean2(raw.resolvedUrl) || classified.resolvedUrl,
+    linkType: raw.linkType ?? classified.linkType,
+    resolutionStatus: raw.resolutionStatus ?? classified.resolutionStatus,
+    title: clean2(raw.title) || null,
+    site: clean2(raw.site) || null
+  };
+}
+function mergeSources(...groups) {
+  const byRawUrl = /* @__PURE__ */ new Map();
+  for (const value of groups.flat()) {
+    const source = normalizeSource(value);
+    if (!source) continue;
+    const existing = byRawUrl.get(source.rawUrl);
+    if (!existing || source.resolutionStatus === "resolved" && existing.resolutionStatus !== "resolved") {
+      byRawUrl.set(source.rawUrl, source);
+    } else {
+      byRawUrl.set(source.rawUrl, { ...existing, title: existing.title ?? source.title, site: existing.site ?? source.site });
+    }
+  }
+  return [...byRawUrl.values()];
+}
 function durableRecordFromFlat(value) {
   if (!value || typeof value !== "object") return null;
   const row = value;
   const question = clean2(row.question);
   if (!normalizePaaQuestion(question)) return null;
-  const sourceCite = clean2(row.source_cite);
-  const sourceTitle = clean2(row.source_title);
-  const sourceSite = clean2(row.source_site);
-  return {
+  const durable = value;
+  const sourceCite = clean2(row.source_cite) || clean2(durable.sourceCite);
+  const sourceTitle = clean2(row.source_title) || clean2(durable.sourceTitle);
+  const sourceSite = clean2(row.source_site) || clean2(durable.sourceSite);
+  const legacySource = sourceCite ? normalizeSource({ title: sourceTitle, site: sourceSite, url: sourceCite, rawUrl: sourceCite }) : null;
+  const sources = mergeSources(Array.isArray(row.sources) ? row.sources : [], legacySource ? [legacySource] : []);
+  const primary = sources[0];
+  return {
+    questionId: clean2(row.question_id) || clean2(durable.questionId) || questionIdFor(question),
+    googleLinkId: clean2(row.google_link_id) || clean2(durable.googleLinkId) || null,
+    googleEvidenceId: clean2(row.google_evidence_id) || clean2(durable.googleEvidenceId) || null,
     question,
     answer: clean2(row.answer),
-    sourceTitle,
-    sourceSite,
-    sourceCite,
-    sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
+    sourceTitle: primary?.title ?? sourceTitle,
+    sourceSite: primary?.site ?? sourceSite,
+    sourceCite: primary?.url ?? sourceCite,
+    sources: mergeSources(sources, Array.isArray(durable.sources) ? durable.sources : []),
     depth: typeof row.depth === "number" && Number.isFinite(row.depth) ? row.depth : 1,
     parentQuestion: clean2(row.parent_question) || null
   };
 }
 function completenessRank(record2) {
   const answer = Boolean(record2.answer.trim());
-  const source = Boolean(record2.sourceCite.trim());
+  const source = record2.sources.length > 0 || Boolean(record2.sourceCite.trim());
   return answer && source ? 3 : answer ? 2 : source ? 1 : 0;
 }
 function sameRecord(left, right) {
-  return left.question === right.question && left.answer === right.answer && left.sourceTitle === right.sourceTitle && left.sourceSite === right.sourceSite && left.sourceCite === right.sourceCite && left.depth === right.depth && left.parentQuestion === right.parentQuestion;
+  return left.questionId === right.questionId && left.googleLinkId === right.googleLinkId && left.googleEvidenceId === right.googleEvidenceId && left.question === right.question && left.answer === right.answer && left.sourceTitle === right.sourceTitle && left.sourceSite === right.sourceSite && left.sourceCite === right.sourceCite && left.depth === right.depth && left.parentQuestion === right.parentQuestion && JSON.stringify(left.sources) === JSON.stringify(right.sources);
 }
 function fillMissing(primary, fallback) {
-  const sourceCite = primary.sourceCite || fallback.sourceCite;
-  const sourceTitle = primary.sourceTitle || fallback.sourceTitle;
-  const sourceSite = primary.sourceSite || fallback.sourceSite;
+  const sources = mergeSources(primary.sources, fallback.sources);
+  const primarySource = sources[0];
+  const sourceCite = primarySource?.url || primary.sourceCite || fallback.sourceCite;
+  const sourceTitle = primarySource?.title || primary.sourceTitle || fallback.sourceTitle;
+  const sourceSite = primarySource?.site || primary.sourceSite || fallback.sourceSite;
   return {
     ...primary,
+    questionId: primary.questionId || fallback.questionId,
+    googleLinkId: primary.googleLinkId || fallback.googleLinkId,
+    googleEvidenceId: primary.googleEvidenceId || fallback.googleEvidenceId,
     question: primary.question || fallback.question,
     answer: primary.answer || fallback.answer,
     sourceTitle,
     sourceSite,
     sourceCite,
-    sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
+    sources,
     depth: Number.isFinite(primary.depth) ? primary.depth : fallback.depth,
     parentQuestion: primary.parentQuestion || fallback.parentQuestion
   };
@@ -26003,23 +26052,28 @@ function mergePaaProgressRecord(existing, incoming, attempt, observedAt) {
 function flatFromRecord(record2, seedQuery) {
   return {
     seed_query: seedQuery,
+    question_id: record2.questionId,
+    google_link_id: record2.googleLinkId ?? "",
+    google_evidence_id: record2.googleEvidenceId ?? "",
     question: record2.question,
     answer: record2.answer,
     source_title: record2.sourceTitle,
     source_site: record2.sourceSite,
     source_cite: record2.sourceCite,
+    sources: record2.sources,
     depth: record2.depth,
     parent_question: record2.parentQuestion ?? "",
     extracted_at: record2.observedAt
   };
 }
 var ProgressivePaaCheckpoint = class {
-  constructor(jobId2, durableMetadata, persist = checkpointJob, now = Date.now) {
+  constructor(jobId2, durableMetadata, persist, now = Date.now, initialResult) {
     this.jobId = jobId2;
     this.durableMetadata = durableMetadata;
     this.persist = persist;
     this.now = now;
     this.lastPersistedAt = now();
+    this.hydrate(initialResult);
   }
   jobId;
   durableMetadata;
@@ -26028,10 +26082,34 @@ var ProgressivePaaCheckpoint = class {
   records = /* @__PURE__ */ new Map();
   clickedQuestions = /* @__PURE__ */ new Set();
   interruptions = [];
+  lifecycle = null;
+  material = null;
   sequence = 0;
   attempt = 1;
   improvementsSinceWrite = 0;
   lastPersistedAt;
+  hydrate(initialResult) {
+    if (!initialResult || typeof initialResult !== "object") return;
+    const progress = initialResult.progress;
+    if (!progress || typeof progress !== "object") return;
+    const value = progress;
+    this.sequence = Number.isFinite(value.sequence) ? Math.max(0, Number(value.sequence)) : 0;
+    this.attempt = Number.isFinite(value.attempt) ? Math.max(1, Number(value.attempt)) : 1;
+    for (const question of value.clickedQuestions ?? []) if (clean2(question)) this.clickedQuestions.add(clean2(question));
+    for (const interruption of value.interruptions ?? []) this.interruptions.push(interruption);
+    if (value.lifecycle) this.lifecycle = { ...value.lifecycle };
+    if (value.material) this.material = structuredClone(value.material);
+    for (const raw of value.records ?? []) {
+      const incoming = durableRecordFromFlat(raw);
+      if (!incoming) continue;
+      const record2 = raw;
+      this.records.set(normalizePaaQuestion(incoming.question), {
+        ...incoming,
+        attempts: Array.isArray(record2.attempts) ? record2.attempts.filter(Number.isFinite) : [this.attempt],
+        observedAt: clean2(record2.observedAt) || new Date(this.now()).toISOString()
+      });
+    }
+  }
   setAttempt(attempt) {
     this.attempt = Math.max(1, attempt);
   }
@@ -26050,6 +26128,11 @@ var ProgressivePaaCheckpoint = class {
   }
   accept(snapshot) {
     this.sequence += 1;
+    if (snapshot.lifecycle) this.lifecycle = { ...snapshot.lifecycle };
+    if (snapshot.material) {
+      this.material = structuredClone(snapshot.material);
+      this.improvementsSinceWrite += 1;
+    }
     for (const question of snapshot.clickedQuestions) if (clean2(question)) this.clickedQuestions.add(clean2(question));
     for (const raw of snapshot.records) {
       const incoming = durableRecordFromFlat(raw);
@@ -26074,6 +26157,9 @@ var ProgressivePaaCheckpoint = class {
   hasRows() {
     return this.records.size > 0;
   }
+  hasPreservedMaterial() {
+    return this.records.size > 0 || this.material !== null;
+  }
   rows() {
     return [...this.records.values()];
   }
@@ -26089,7 +26175,9 @@ var ProgressivePaaCheckpoint = class {
       captchaObserved: false,
       solverCleared: false,
       error,
-      interruptions: [...this.interruptions]
+      interruptions: [...this.interruptions],
+      lifecycle: this.lifecycle ? { ...this.lifecycle } : null,
+      material: this.material ? structuredClone(this.material) : null
     };
   }
   async flush(phase, options = {}) {
@@ -26106,11 +26194,12 @@ var ProgressivePaaCheckpoint = class {
     return true;
   }
 };
-function partialTerminalResult(job, options, maxQuestions, progress) {
+function partialTerminalResult(job, options, maxQuestions, progress, recoveryCount) {
   const flat = progress.records.map((record2) => flatFromRecord(record2, options.query ?? job.query));
   const withoutAnswer = flat.filter((row) => !row.answer).length;
-  const withoutSource = flat.filter((row) => !row.source_cite).length;
+  const withoutSource = flat.filter((row) => row.sources.length === 0 && !row.source_cite).length;
   const latestInterruption = progress.interruptions.at(-1);
+  const material = progress.material;
   return {
     seed: options.query ?? job.query,
     location: options.location ?? null,
@@ -26118,22 +26207,40 @@ function partialTerminalResult(job, options, maxQuestions, progress) {
     totalQuestions: flat.length,
     tree: [],
     flat,
-    videos: [],
-    forums: [],
-    organicResults: [],
-    localPack: [],
-    whatPeopleSaying: [],
-    aiOverview: { detected: false, text: null, citations: [] },
-    aiMode: { detected: false, text: null, citations: [] },
-    entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
-    surface: "unknown",
+    videos: material?.videos ?? [],
+    forums: material?.forums ?? [],
+    organicResults: material?.organicResults ?? [],
+    localPack: material?.localPack ?? [],
+    whatPeopleSaying: material?.whatPeopleSaying ?? [],
+    aiOverview: material?.aiOverview ?? { detected: false, text: null, citations: [] },
+    aiMode: material?.aiMode ?? { detected: false, text: null, citations: [] },
+    entityIds: material?.entityIds ?? { entities: [], kgIds: [], cids: [], gcids: [] },
+    surface: material?.surface ?? "unknown",
     stats: { seed: options.query ?? job.query, totalQuestions: flat.length, maxDepthReached: 1, durationMs: 0, errorCount: 1 },
     diagnostics: {
       completionStatus: "paa_found",
       problem: null,
       resultQuality: "partial",
       degradedResult: false,
-      retryRecommended: false,
+      retryRecommended: true,
+      paaLifecycle: {
+        ...progress.lifecycle ?? {
+          requestedQuestions: maxQuestions,
+          questionsFound: flat.length,
+          targetStatus: flat.length >= maxQuestions ? "met" : "not_met",
+          materialStatus: "partial",
+          knownQuestions: flat.length,
+          processedQuestions: progress.clickedQuestions.length,
+          failedInteractions: 0,
+          noGrowthConfirmations: 0
+        },
+        questionsFound: flat.length,
+        targetStatus: flat.length >= maxQuestions ? "met" : "not_met",
+        discoveryStatus: "recovery_exhausted",
+        materialStatus: "partial",
+        completionReason: "Automatic runtime recovery was exhausted; all preserved material is returned.",
+        automaticRetries: recoveryCount + progress.interruptions.filter((value) => value.willRetry).length
+      },
       completeness: {
         paaWithoutAnswer: withoutAnswer,
         paaWithoutSource: withoutSource,
@@ -26143,7 +26250,7 @@ function partialTerminalResult(job, options, maxQuestions, progress) {
         paaReturned: flat.length,
         paaUnique: flat.length
       },
-      warnings: [{ code: "paa_partial", surface: "paa", message: "Provider attempts ended before the requested PAA capture completed.", retryable: false }],
+      warnings: [{ code: "paa_partial", surface: "paa", message: "Automatic recovery ended before frontier exhaustion could be proven; preserved material is returned.", retryable: true }],
       ...latestInterruption && ["browser_session_interrupted", "request_aborted", "timeout"].includes(latestInterruption.outcome) ? { interruption: { code: latestInterruption.outcome, reasonSource: latestInterruption.reasonSource, message: null } } : {}
     },
     progress,
@@ -26171,21 +26278,36 @@ async function settleCompletedJob(jobId2) {
 }
 async function runPaaHarvestJob(jobId2) {
   const invocationStartedAt = Date.now();
-  const job = await claimDurablePaaJob(jobId2);
-  if (!job) return { jobId: jobId2, status: "missing_or_not_paa" };
+  const claim = await claimDurablePaaJob(jobId2);
+  if (!claim) return { jobId: jobId2, status: "missing_or_not_paa" };
+  const { job, leaseToken, recoveryCount } = claim;
   if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
     await settleCompletedJob(job.id);
     return { jobId: job.id, status: job.status };
   }
+  if (!leaseToken) return { jobId: job.id, status: "missing_lease" };
   const options = job.options;
   const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
   const timeoutBudget = harvestTimeoutBudget(maxQuestions);
   const workBudgetMs = Math.max(1, timeoutBudget.serverMs - CLEANUP_RESERVE_MS);
-  const progress = new ProgressivePaaCheckpoint(job.id, {
-    invocationBudgetMs: timeoutBudget.serverMs,
-    workBudgetMs,
-    cleanupReserveMs: CLEANUP_RESERVE_MS
-  });
+  const progress = new ProgressivePaaCheckpoint(
+    job.id,
+    {
+      invocationBudgetMs: timeoutBudget.serverMs,
+      workBudgetMs,
+      cleanupReserveMs: CLEANUP_RESERVE_MS,
+      recoveryCount
+    },
+    (_id, result2) => checkpointDurablePaaJob(job.id, leaseToken, result2),
+    Date.now,
+    job.result
+  );
+  const heartbeat = setInterval(() => {
+    void renewDurablePaaLease(job.id, leaseToken).catch((error) => {
+      console.error("[paa-harvest] lease heartbeat failed:", error instanceof Error ? error.message : String(error));
+    });
+  }, LEASE_HEARTBEAT_MS);
+  heartbeat.unref?.();
   try {
     if (!brightDataSerpEnabled()) throw new Error("SERP browser service is not configured");
     await progress.flush("navigating", { force: true });
@@ -26238,8 +26360,12 @@ async function runPaaHarvestJob(jobId2) {
     const rawDiagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : {};
     const rawCompleteness = rawDiagnostics.completeness && typeof rawDiagnostics.completeness === "object" ? rawDiagnostics.completeness : {};
     const missingAnswers = mergedFlat.filter((row) => !row.answer).length;
-    const missingSources = mergedFlat.filter((row) => !row.source_cite).length;
-    const incomplete = mergedFlat.length > 0 && (mergedFlat.length < maxQuestions || missingAnswers > 0 || missingSources > 0);
+    const missingSources = mergedFlat.filter((row) => row.sources.length === 0 && !row.source_cite).length;
+    const rawLifecycle = rawDiagnostics.paaLifecycle && typeof rawDiagnostics.paaLifecycle === "object" ? rawDiagnostics.paaLifecycle : {};
+    const discoveryStatus = typeof rawLifecycle.discoveryStatus === "string" ? rawLifecycle.discoveryStatus : "interrupted";
+    const materialPartial = missingAnswers > 0 || missingSources > 0;
+    const incomplete = discoveryStatus === "interrupted" || discoveryStatus === "recovery_exhausted" || materialPartial;
+    const automaticRetries = recoveryCount + terminalProgress.interruptions.filter((value) => value.willRetry).length;
     const result2 = {
       ...harvestResult,
       totalQuestions: mergedFlat.length,
@@ -26247,7 +26373,17 @@ async function runPaaHarvestJob(jobId2) {
       stats: harvestResult.stats && typeof harvestResult.stats === "object" ? { ...harvestResult.stats, totalQuestions: mergedFlat.length } : harvestResult.stats,
       diagnostics: {
         ...rawDiagnostics,
-        ...incomplete ? { resultQuality: "partial", degradedResult: false, retryRecommended: true } : {},
+        resultQuality: incomplete ? "partial" : "complete",
+        degradedResult: false,
+        retryRecommended: discoveryStatus === "interrupted" || discoveryStatus === "recovery_exhausted",
+        paaLifecycle: {
+          ...rawLifecycle,
+          requestedQuestions: maxQuestions,
+          questionsFound: mergedFlat.length,
+          targetStatus: mergedFlat.length >= maxQuestions ? "met" : "not_met",
+          materialStatus: materialPartial ? "partial" : "complete",
+          automaticRetries
+        },
         completeness: {
           ...rawCompleteness,
           paaWithoutAnswer: missingAnswers,
@@ -26267,7 +26403,7 @@ async function runPaaHarvestJob(jobId2) {
         rawDomArtifactStatus: "unavailable"
       }
     };
-    if (!await completeRunningJob(job.id, result2)) throw new Error("job_left_running_state_during_capture");
+    if (!await completeDurablePaaJob(job.id, leaseToken, result2)) throw new Error("job_left_running_state_during_capture");
     await settleCompletedJob(job.id);
     if (job.callback_url) await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result: result2 });
     return { jobId: job.id, status: "done", totalQuestions: questionCount(result2), partial: incomplete, artifactStored: false };
@@ -26276,16 +26412,16 @@ async function runPaaHarvestJob(jobId2) {
     const privateError = serializeHarvestProblem(problem);
     await progress.flush("failed", { force: true, error: privateError }).catch(() => {
     });
-    if (progress.hasRows()) {
+    if (progress.hasPreservedMaterial()) {
       const terminalProgress = progress.checkpoint("failed", privateError);
-      const result2 = partialTerminalResult(job, options, maxQuestions, terminalProgress);
-      if (await completeRunningJob(job.id, result2)) {
+      const result2 = partialTerminalResult(job, options, maxQuestions, terminalProgress, recoveryCount);
+      if (await completeDurablePaaJob(job.id, leaseToken, result2)) {
         await settleCompletedJob(job.id);
         if (job.callback_url) await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result: result2 });
         return { jobId: job.id, status: "done", totalQuestions: questionCount(result2), partial: true, artifactStored: false };
       }
     }
-    await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
+    await failDurablePaaJob(job.id, leaseToken, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
     await settlePaaHarvestJob(job.id).then(async (settled) => {
       if (!settled) return;
       const settledJob = await getJob(job.id);
@@ -26303,6 +26439,8 @@ async function runPaaHarvestJob(jobId2) {
       });
     }
     throw error;
+  } finally {
+    clearInterval(heartbeat);
   }
 }
 
@@ -26501,22 +26639,48 @@ function recoverableRecords(result2) {
       questions.set(question, record2);
       continue;
     }
-    const completeness = (value) => Number(Boolean(String(value.answer ?? "").trim())) * 2 + Number(Boolean(String(value.sourceCite ?? value.source_cite ?? "").trim()));
-    if (completeness(record2) >= completeness(existing)) questions.set(question, record2);
+    const sourcesOf = (value) => Array.isArray(value.sources) ? value.sources.filter((source) => source && typeof source === "object") : [];
+    const completeness = (value) => {
+      const sources = sourcesOf(value);
+      return Number(Boolean(String(value.answer ?? "").trim())) * 4 + Number(sources.length > 0 || Boolean(String(value.sourceCite ?? value.source_cite ?? "").trim())) * 2 + sources.filter((source) => source.resolutionStatus === "resolved").length;
+    };
+    const preferred = completeness(record2) >= completeness(existing) ? record2 : existing;
+    const fallback = preferred === record2 ? existing : record2;
+    const mergedSources = /* @__PURE__ */ new Map();
+    for (const source of [...sourcesOf(fallback), ...sourcesOf(preferred)]) {
+      const key2 = String(source.rawUrl ?? source.url ?? "");
+      if (!key2) continue;
+      const current = mergedSources.get(key2);
+      if (!current || source.resolutionStatus === "resolved") mergedSources.set(key2, source);
+    }
+    questions.set(question, {
+      ...fallback,
+      ...preferred,
+      questionId: String(preferred.questionId ?? fallback.questionId ?? questionIdFor(String(preferred.question ?? fallback.question ?? ""))),
+      googleLinkId: preferred.googleLinkId ?? fallback.googleLinkId ?? null,
+      googleEvidenceId: preferred.googleEvidenceId ?? fallback.googleEvidenceId ?? null,
+      sources: [...mergedSources.values()]
+    });
   }
   return [...questions.values()];
 }
-function stalePartialResult(result2, records, seedQuery) {
+function stalePartialResult(result2, records, seedQuery, recoveryCount) {
   const current = result2 && typeof result2 === "object" ? result2 : {};
+  const progress = current.progress && typeof current.progress === "object" ? current.progress : {};
+  const material = progress.material && typeof progress.material === "object" ? progress.material : {};
   const diagnostics = current.diagnostics && typeof current.diagnostics === "object" ? current.diagnostics : {};
   const completeness = diagnostics.completeness && typeof diagnostics.completeness === "object" ? diagnostics.completeness : {};
   const flat = records.map((record2) => ({
     seed_query: typeof record2.seed_query === "string" ? record2.seed_query : seedQuery,
+    question_id: typeof record2.question_id === "string" ? record2.question_id : typeof record2.questionId === "string" ? record2.questionId : questionIdFor(String(record2.question ?? "")),
+    google_link_id: typeof record2.google_link_id === "string" ? record2.google_link_id : typeof record2.googleLinkId === "string" ? record2.googleLinkId : "",
+    google_evidence_id: typeof record2.google_evidence_id === "string" ? record2.google_evidence_id : typeof record2.googleEvidenceId === "string" ? record2.googleEvidenceId : "",
     question: typeof record2.question === "string" ? record2.question : "",
     answer: typeof record2.answer === "string" ? record2.answer : "",
     source_title: typeof record2.source_title === "string" ? record2.source_title : typeof record2.sourceTitle === "string" ? record2.sourceTitle : "",
     source_site: typeof record2.source_site === "string" ? record2.source_site : typeof record2.sourceSite === "string" ? record2.sourceSite : "",
     source_cite: typeof record2.source_cite === "string" ? record2.source_cite : typeof record2.sourceCite === "string" ? record2.sourceCite : "",
+    sources: Array.isArray(record2.sources) ? record2.sources : [],
     depth: typeof record2.depth === "number" ? record2.depth : 1,
     parent_question: typeof record2.parent_question === "string" ? record2.parent_question : typeof record2.parentQuestion === "string" ? record2.parentQuestion : "",
     extracted_at: typeof record2.extracted_at === "string" ? record2.extracted_at : typeof record2.observedAt === "string" ? record2.observedAt : (/* @__PURE__ */ new Date()).toISOString()
@@ -26527,6 +26691,15 @@ function stalePartialResult(result2, records, seedQuery) {
     totalQuestions: uniqueCount,
     flat,
     tree: Array.isArray(current.tree) ? current.tree : [],
+    videos: Array.isArray(material.videos) ? material.videos : current.videos ?? [],
+    forums: Array.isArray(material.forums) ? material.forums : current.forums ?? [],
+    organicResults: Array.isArray(material.organicResults) ? material.organicResults : current.organicResults ?? [],
+    localPack: Array.isArray(material.localPack) ? material.localPack : current.localPack ?? [],
+    whatPeopleSaying: Array.isArray(material.whatPeopleSaying) ? material.whatPeopleSaying : current.whatPeopleSaying ?? [],
+    aiOverview: material.aiOverview ?? current.aiOverview,
+    aiMode: material.aiMode ?? current.aiMode,
+    entityIds: material.entityIds ?? current.entityIds,
+    surface: material.surface ?? current.surface,
     partial: true,
     diagnostics: {
       ...diagnostics,
@@ -26534,7 +26707,17 @@ function stalePartialResult(result2, records, seedQuery) {
       problem: null,
       resultQuality: "partial",
       degradedResult: false,
-      retryRecommended: false,
+      retryRecommended: true,
+      paaLifecycle: {
+        ...diagnostics.paaLifecycle && typeof diagnostics.paaLifecycle === "object" ? diagnostics.paaLifecycle : {},
+        requestedQuestions: Number(completeness.paaRequested ?? uniqueCount),
+        questionsFound: uniqueCount,
+        targetStatus: uniqueCount >= Number(completeness.paaRequested ?? Number.MAX_SAFE_INTEGER) ? "met" : "not_met",
+        discoveryStatus: "recovery_exhausted",
+        materialStatus: "partial",
+        completionReason: "Automatic runtime recovery was exhausted; all preserved material is returned.",
+        automaticRetries: recoveryCount
+      },
       interruption: {
         code: "timeout",
         reasonSource: "client_runtime",
@@ -26553,7 +26736,16 @@ function stalePartialResult(result2, records, seedQuery) {
     }
   };
 }
-async function reconcilePaaHarvestSettlements(limit = 25) {
+var MAX_RUNTIME_RECOVERIES = 2;
+async function dispatchRecovery(origin, jobId2) {
+  if (!process.env.CRON_SECRET) return false;
+  const response = await fetch(`${origin}/harvest/internal/resume/${encodeURIComponent(jobId2)}`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
+  });
+  return response.ok || response.status === 409;
+}
+async function reconcilePaaHarvestSettlements(limit = 25, origin) {
   let providerChecked = 0;
   let providerCompleted = 0;
   let providerFailed = 0;
@@ -26565,19 +26757,32 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
   } catch {
     providerFailed = 1;
   }
-  const stale = await listStaleRunningDurablePaaJobs(limit);
+  const stale = await listRecoverableDurablePaaJobs(limit);
   let stalePartial = 0;
   let staleFailed = 0;
+  let requeued = 0;
+  let redispatched = 0;
   for (const job of stale) {
     try {
+      const recoveryCount = Math.max(0, Number(job.options.paaRecoveryCount ?? 0));
+      if (job.status === "pending") {
+        if (origin && await dispatchRecovery(origin, job.id)) redispatched += 1;
+        continue;
+      }
+      if (recoveryCount < MAX_RUNTIME_RECOVERIES && await requeueExpiredDurablePaaJob(job.id, MAX_RUNTIME_RECOVERIES)) {
+        requeued += 1;
+        if (origin && await dispatchRecovery(origin, job.id)) redispatched += 1;
+        continue;
+      }
       const retained = recoverableRecords(job.result);
-      if (retained.length > 0) {
-        if (await completeRunningJob(job.id, stalePartialResult(job.result, retained, job.query))) stalePartial += 1;
+      const progress = job.result && typeof job.result === "object" ? job.result.progress : void 0;
+      if (retained.length > 0 || Boolean(progress?.material)) {
+        if (await completeExpiredDurablePaaJob(job.id, stalePartialResult(job.result, retained, job.query, recoveryCount))) stalePartial += 1;
         continue;
       }
-      const privateError = "direct PAA invocation exceeded its platform safety window; no recoverable rows were retained";
+      const privateError = "PAA runtime recovery was exhausted; no recoverable rows were retained";
       const problem = classifyHarvestProblem(new Error("harvest timeout"));
-      if (await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
+      if (await failExpiredDurablePaaJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
         staleFailed += 1;
       }
     } catch {
@@ -26607,6 +26812,8 @@ async function reconcilePaaHarvestSettlements(limit = 25) {
     checked: jobs.length,
     stalePartial,
     staleFailed,
+    requeued,
+    redispatched,
     settled,
     failed,
     providerChecked,
@@ -42270,7 +42477,7 @@ async function prepareCommonsEntity(input, user) {
   const duplicateCandidates = await potentialDuplicateCandidates({
     title,
     slug: slug4,
-    source: normalizeSource(input.source),
+    source: normalizeSource2(input.source),
     maxCandidates: clampNumber(input.maxCandidates, 1, 20, 8)
   });
   const tagVocabulary = await commonsTagVocabulary();
@@ -42324,7 +42531,7 @@ async function validateCommonsEntity(input, user) {
   const description = optionalText2(input.description, 600);
   const slug4 = normalizeSlug(input.slug || title || "");
   const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
-  const source = normalizeSource(input.source);
+  const source = normalizeSource2(input.source);
   const featuredImage = normalizeFeaturedImage(input.featuredImage);
   const normalizedFeaturedImageUrl = featuredImage?.url ? normalizeCommonsImageUrl(featuredImage.url) : null;
   const registeredFeaturedImage = normalizedFeaturedImageUrl ? await findCommonsHostedImageByUrl(normalizedFeaturedImageUrl) : null;
@@ -43066,7 +43273,7 @@ function normalizeSubmitInput(input) {
     entityType: normalizeEntityType(input.entityType),
     disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
     featuredImage: normalizeFeaturedImage(input.featuredImage),
-    source: normalizeSource(input.source),
+    source: normalizeSource2(input.source),
     tags: uniqueStrings(input.tags ?? []).slice(0, 24),
     keywords: uniqueStrings(input.keywords ?? []).slice(0, 50),
     relatedEntities: normalizeRelatedEntities(input.relatedEntities),
@@ -43687,7 +43894,7 @@ function normalizeMedia(media, featuredImage) {
     return true;
   }).slice(0, 300);
 }
-function normalizeSource(source) {
+function normalizeSource2(source) {
   return {
     originalUrl: optionalText2(source?.originalUrl, 2e3) ?? void 0,
     resolvedUrl: optionalText2(source?.resolvedUrl, 2e3) ?? void 0,
@@ -47156,7 +47363,7 @@ async function callMainOwnedExportPage(identity, input) {
     });
   }
   if (connection.providerConfigKey === "google-mail" && input.dataset === "emails") {
-    const { GmailService: GmailService2 } = await import("./gmail-service-TIU4V5JA.js");
+    const { GmailService: GmailService2 } = await import("./gmail-service-HN57R2GS.js");
     const service = new GmailService2();
     const ownerId3 = createHash29("sha256").update(identity.toLowerCase()).digest("hex").slice(0, 24);
     const after = Math.floor(Date.parse(input.from) / 1e3);
@@ -61115,6 +61322,19 @@ async function checkHarvestLimits(user, reuseLockId) {
   }
   return null;
 }
+app.post("/harvest/internal/resume/:id", async (c) => {
+  const secret = c.req.header("authorization");
+  if (!process.env.CRON_SECRET || secret !== `Bearer ${process.env.CRON_SECRET}`) {
+    return c.json({ error: "Unauthorized" }, 401);
+  }
+  const job = await getJob(c.req.param("id"));
+  if (!job || job.options.executionOwner !== "direct" || job.options.serpOnly === true) {
+    return c.json({ error: "PAA job not found" }, 404);
+  }
+  if (job.status !== "pending") return c.json({ job_id: job.id, status: job.status }, 409);
+  scheduleDirectPaaHarvest(job.id);
+  return c.json({ job_id: job.id, status: "pending", recovery_dispatched: true }, 202);
+});
 app.post("/harvest", auth5, async (c) => {
   const user = c.get("user");
   const raw = await c.req.json().catch(() => ({}));
@@ -61198,6 +61418,7 @@ app.post("/harvest", auth5, async (c) => {
       await createJobWithId(jobId2, user.id, options.query, {
         ...options,
         executionOwner: "direct",
+        paaRecoveryCount: 0,
         billingHoldMc: harvestCost,
         billingDebitKey,
         ...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
@@ -62737,7 +62958,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-DR6R7J46.js");
+  const { drainQueue } = await import("./worker-2BYACLNF.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];
@@ -62775,10 +62996,12 @@ app.get("/cron/tick", async (c) => {
     settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
     settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
     settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
-    settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), {
+    settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10, origin), {
       checked: 0,
       stalePartial: 0,
       staleFailed: 0,
+      requeued: 0,
+      redispatched: 0,
       settled: 0,
       failed: 0,
       providerChecked: 0,
@@ -62859,8 +63082,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     finishExtractJob: finishExtractJob2,
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
-  } = await import("./site-extract-repository-B47JB62U.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-OO74L67T.js");
+  } = await import("./site-extract-repository-XKSVXSSY.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-P2DTUVY3.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -62887,7 +63110,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-B47JB62U.js");
+    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-XKSVXSSY.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId2);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);