mcp-scraper 0.79.2 → 0.79.3

package/dist/{server-5OSHEEVT.js → server-MDWFOUTS.js}

@@ -10,7 +10,7 @@ import {
   queuedCommonsDocumentCount,
   semanticCommonsEntityScores,
   typeFromMime
-} from "./chunk-BTQ23KCT.js";
+} from "./chunk-F5GQJWZU.js";
 import {
   SITE_EXTRACT_ARTIFACT_PREFIX,
   abandonExtractSettlement,
@@ -41,13 +41,13 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-SL7YVA4S.js";
+} from "./chunk-YBR32LOD.js";
 import {
   REQUIRED_LOCATION_STATE_CODES,
   importHostedCensusPlacesCsv,
   importHostedZipGroupsCsv,
   listActiveHostedLocationDatasets
-} from "./chunk-OUBH6XMY.js";
+} from "./chunk-K3Z5AQYE.js";
 import {
   cancelPendingLeadListEnrichmentRows,
   claimLeadListEnrichmentOutbox,
@@ -74,7 +74,7 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-KAGHJQFO.js";
+} from "./chunk-EQGTEHLZ.js";
 import {
   ConnectedUsageBillingError,
   ConnectedUsagePreflightInputSchema,
@@ -101,7 +101,7 @@ import {
   probeNangoConnectionDirect,
   recordNangoConnectionCost,
   settleConnectedUsage
-} from "./chunk-LB2GNACG.js";
+} from "./chunk-3423R23T.js";
 import {
   ANALYTICS_CONTENT_SORTS,
   AnalyticsIdentityProfileError,
@@ -185,7 +185,7 @@ import {
   upsertAnalyticsAdSpend,
   upsertAnalyticsHostGroup,
   verifyAnalyticsWebhookSignature
-} from "./chunk-DUWZQSSJ.js";
+} from "./chunk-HEUMDX4I.js";
 import {
   claimServiceConnectionAction,
   ensureServiceConnectionsSchema,
@@ -198,7 +198,7 @@ import {
   setServiceConnectionUserLabel,
   signSession,
   verifySession
-} from "./chunk-Q5L2OCOQ.js";
+} from "./chunk-C5Z4OFKW.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
@@ -207,7 +207,7 @@ import {
   harvestProblemResponse,
   publicErrorBoundary,
   serializeHarvestProblem
-} from "./chunk-47ZCUBM5.js";
+} from "./chunk-NVUKO5NN.js";
 import "./chunk-OZJMVCDK.js";
 import {
   BACKUP_LAST_RESORT_ATTEMPTS,
@@ -229,7 +229,7 @@ import {
   serpArrivalUrl,
   validateProviderSessionId,
   withRuntimeUserAgent
-} from "./chunk-WEEAXPWK.js";
+} from "./chunk-KNGTQD6U.js";
 import {
   AnalyticsConfirmedPersonProjectionSchema,
   AnalyticsCrmFieldSchema,
@@ -316,7 +316,7 @@ import {
   resolveDeploymentProfile,
   resolveLocalSourcebookSchemaType,
   transcribeMediaUrl
-} from "./chunk-JECPQBLZ.js";
+} from "./chunk-7UZAVM4J.js";
 import {
   auditImageUrls,
   auditImages,
@@ -340,7 +340,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   sanitizeOutboundDiagnostics
-} from "./chunk-B6L73AO4.js";
+} from "./chunk-6HAV7LCE.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PACK_INVOICE_DESCRIPTION,
@@ -399,7 +399,7 @@ import {
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-NGLZ6UJR.js";
+} from "./chunk-5UAFBKJR.js";
 import {
   listWorkflowDefinitions,
   runWorkflow,
@@ -435,7 +435,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2RTELL6M.js";
+} from "./chunk-IHYG52ZW.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -476,6 +476,7 @@ import {
   failJob,
   failStripeEvent,
   failWorkflowRunRecord,
+  finalizeExpiredHarvestAttempts,
   getBillingLedgerSummary,
   getClient,
   getDb,
@@ -546,7 +547,7 @@ import {
   updateHarvestAttemptProviderTelemetry,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -9647,7 +9648,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-I2GNC25L.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-S3ZGNFML.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10607,7 +10608,7 @@ async function resolveUsZipsPath(requestedPath) {
 async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
   const mode = resolvedLocationDataMode({ zipSource: sourceMode });
   if (mode === "hosted") {
-    const { getHostedZipGroups } = await import("./location-data-repository-AHKDXTEQ.js");
+    const { getHostedZipGroups } = await import("./location-data-repository-WPRG62GE.js");
     const hosted = await getHostedZipGroups(stateAbbr);
     if (!hosted.dataset) {
       throw new DirectoryLocationError(
@@ -10670,7 +10671,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
   const sourceUrl = censusStateUrl(state2.fips);
   const warnings = [];
   if (resolvedLocationDataMode(config) === "hosted") {
-    const { queryHostedLocationMarkets } = await import("./location-data-repository-AHKDXTEQ.js");
+    const { queryHostedLocationMarkets } = await import("./location-data-repository-WPRG62GE.js");
     const hosted = await queryHostedLocationMarkets({
       state: state2.abbr,
       minPopulation: options.minPopulation,
@@ -12157,7 +12158,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
 async function settlePersistedLeadListEnrichment(jobId2) {
   const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
   const job = jobs.find((candidate) => candidate.id === jobId2);
-  const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-5LEWW4X7.js");
+  const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-36RPVV6N.js");
   const current = job ?? await getLeadListEnrichmentJob2(jobId2);
   if (!current) return null;
   const rows = await listLeadListEnrichmentRows(jobId2);
@@ -12213,7 +12214,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
         const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
         const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
         if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
-          const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-5LEWW4X7.js");
+          const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-36RPVV6N.js");
           await failLeadListEnrichmentJob2({
             jobId: job.id,
             errorCode: "lead_list_all_rows_failed",
@@ -25946,6 +25947,25 @@ var DIRECT_PAA_MAX_ATTEMPTS = 3;
 var CHECKPOINT_INTERVAL_MS = 1e4;
 var CHECKPOINT_IMPROVEMENT_THRESHOLD = 5;
 var LEASE_HEARTBEAT_MS = 1e4;
+var MAX_DURABLE_INTERACTION_OBSERVATIONS = 250;
+function summarizePaaInteractions(observations) {
+  const hasSignal = (value, signal) => value.confirmationSignals.includes(signal);
+  return {
+    observations: observations.length,
+    dispatched: observations.filter((value) => value.dispatched).length,
+    confirmed: observations.filter((value) => value.outcome === "confirmed").length,
+    unconfirmed: observations.filter((value) => value.outcome === "unconfirmed").length,
+    dispatchFailures: observations.filter((value) => value.outcome === "dispatch_failed").length,
+    confirmedWithoutDispatchAck: observations.filter((value) => value.outcome === "confirmed" && !value.dispatched).length,
+    expandedSignals: observations.filter((value) => hasSignal(value, "expanded")).length,
+    answerSignals: observations.filter((value) => hasSignal(value, "answer")).length,
+    sourceSignals: observations.filter((value) => hasSignal(value, "source")).length,
+    questionGrowthBatches: new Set(observations.filter((value) => hasSignal(value, "question_growth")).map((value) => value.batchNumber)).size,
+    checkpoint700ms: observations.filter((value) => value.confirmationCheckpointMs === 700).length,
+    checkpoint1000ms: observations.filter((value) => value.confirmationCheckpointMs === 1e3).length,
+    checkpoint1400ms: observations.filter((value) => value.confirmationCheckpointMs === 1400).length
+  };
+}
 function normalizePaaQuestion(value) {
   if (typeof value !== "string") return "";
   return value.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
@@ -26084,6 +26104,7 @@ var ProgressivePaaCheckpoint = class {
   interruptions = [];
   lifecycle = null;
   material = null;
+  interactions = /* @__PURE__ */ new Map();
   sequence = 0;
   attempt = 1;
   improvementsSinceWrite = 0;
@@ -26099,6 +26120,10 @@ var ProgressivePaaCheckpoint = class {
     for (const interruption of value.interruptions ?? []) this.interruptions.push(interruption);
     if (value.lifecycle) this.lifecycle = { ...value.lifecycle };
     if (value.material) this.material = structuredClone(value.material);
+    for (const interaction of value.interactions ?? []) {
+      if (!interaction?.id) continue;
+      this.interactions.set(interaction.id, structuredClone(interaction));
+    }
     for (const raw of value.records ?? []) {
       const incoming = durableRecordFromFlat(raw);
       if (!incoming) continue;
@@ -26134,6 +26159,15 @@ var ProgressivePaaCheckpoint = class {
       this.improvementsSinceWrite += 1;
     }
     for (const question of snapshot.clickedQuestions) if (clean2(question)) this.clickedQuestions.add(clean2(question));
+    for (const interaction of snapshot.interactions ?? []) {
+      if (!interaction?.id) continue;
+      this.interactions.set(interaction.id, structuredClone(interaction));
+    }
+    while (this.interactions.size > MAX_DURABLE_INTERACTION_OBSERVATIONS) {
+      const oldest = this.interactions.keys().next().value;
+      if (oldest === void 0) break;
+      this.interactions.delete(oldest);
+    }
     for (const raw of snapshot.records) {
       const incoming = durableRecordFromFlat(raw);
       if (!incoming) continue;
@@ -26164,6 +26198,7 @@ var ProgressivePaaCheckpoint = class {
     return [...this.records.values()];
   }
   checkpoint(phase, error = null) {
+    const interactions = [...this.interactions.values()];
     return {
       phase,
       attempt: this.attempt,
@@ -26177,7 +26212,9 @@ var ProgressivePaaCheckpoint = class {
       error,
       interruptions: [...this.interruptions],
       lifecycle: this.lifecycle ? { ...this.lifecycle } : null,
-      material: this.material ? structuredClone(this.material) : null
+      material: this.material ? structuredClone(this.material) : null,
+      interactions,
+      interactionSummary: summarizePaaInteractions(interactions)
     };
   }
   async flush(phase, options = {}) {
@@ -26324,7 +26361,8 @@ async function runPaaHarvestJob(jobId2) {
     };
     const onPaaProgress = async (snapshot) => {
       progress.accept(snapshot);
-      await progress.flush(snapshot.phase === "attempt_ending" ? "failed" : "expanding", { force: snapshot.phase === "attempt_ending" });
+      const force = snapshot.phase === "attempt_ending" || snapshot.phase === "click_observed" || snapshot.phase === "expansion_finished";
+      await progress.flush(snapshot.phase === "attempt_ending" ? "failed" : "expanding", { force });
     };
     const harvestCtx = currentCostContext();
     const harvested = await runWithCostContext(
@@ -26762,6 +26800,7 @@ async function reconcilePaaHarvestSettlements(limit = 25, origin) {
   let staleFailed = 0;
   let requeued = 0;
   let redispatched = 0;
+  let attemptsFinalized = 0;
   for (const job of stale) {
     try {
       const recoveryCount = Math.max(0, Number(job.options.paaRecoveryCount ?? 0));
@@ -26769,7 +26808,12 @@ async function reconcilePaaHarvestSettlements(limit = 25, origin) {
         if (origin && await dispatchRecovery(origin, job.id)) redispatched += 1;
         continue;
       }
-      if (recoveryCount < MAX_RUNTIME_RECOVERIES && await requeueExpiredDurablePaaJob(job.id, MAX_RUNTIME_RECOVERIES)) {
+      const willRetry = recoveryCount < MAX_RUNTIME_RECOVERIES;
+      try {
+        attemptsFinalized += await finalizeExpiredHarvestAttempts(job.id, willRetry);
+      } catch {
+      }
+      if (willRetry && await requeueExpiredDurablePaaJob(job.id, MAX_RUNTIME_RECOVERIES)) {
         requeued += 1;
         if (origin && await dispatchRecovery(origin, job.id)) redispatched += 1;
         continue;
@@ -26818,7 +26862,8 @@ async function reconcilePaaHarvestSettlements(limit = 25, origin) {
     failed,
     providerChecked,
     providerCompleted,
-    providerFailed
+    providerFailed,
+    attemptsFinalized
   };
 }
 
@@ -47766,7 +47811,7 @@ async function callMainOwnedExportPage(identity, input) {
     });
   }
   if (connection.providerConfigKey === "google-mail" && input.dataset === "emails") {
-    const { GmailService: GmailService2 } = await import("./gmail-service-GGQVJUC3.js");
+    const { GmailService: GmailService2 } = await import("./gmail-service-YFS2OV4S.js");
     const service = new GmailService2();
     const ownerId3 = createHash29("sha256").update(identity.toLowerCase()).digest("hex").slice(0, 24);
     const after = Math.floor(Date.parse(input.from) / 1e3);
@@ -77738,7 +77783,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-2TYCYFPM.js");
+  const { drainQueue } = await import("./worker-3LUFQWWJ.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];
@@ -77862,8 +77907,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     finishExtractJob: finishExtractJob2,
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
-  } = await import("./site-extract-repository-KF736IR6.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-I2GNC25L.js");
+  } = await import("./site-extract-repository-WAJMIDPJ.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-S3ZGNFML.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -77890,7 +77935,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-KF736IR6.js");
+    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-WAJMIDPJ.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId2);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);

package/dist/{site-extract-repository-KF736IR6.js → site-extract-repository-WAJMIDPJ.js}

@@ -28,10 +28,10 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-SL7YVA4S.js";
+} from "./chunk-YBR32LOD.js";
 import "./chunk-DNM65UCK.js";
 import "./chunk-VFCUGTCF.js";
-import "./chunk-OU4FV2RP.js";
+import "./chunk-YXNDOQXN.js";
 export {
   abandonExtractSettlement,
   claimFailedExtractJobForRefinalize,

package/dist/{worker-2TYCYFPM.js → worker-3LUFQWWJ.js}

@@ -5,15 +5,15 @@ import {
   harvestProblemEnvelope,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-47ZCUBM5.js";
+} from "./chunk-NVUKO5NN.js";
 import "./chunk-OZJMVCDK.js";
 import {
   harvest
-} from "./chunk-WEEAXPWK.js";
+} from "./chunk-KNGTQD6U.js";
 import {
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-B6L73AO4.js";
+} from "./chunk-6HAV7LCE.js";
 import {
   MC_COSTS,
   serpActualCostMc
@@ -23,7 +23,7 @@ import {
 } from "./chunk-OM7HVEJ3.js";
 import {
   runWithCostContext
-} from "./chunk-NGLZ6UJR.js";
+} from "./chunk-5UAFBKJR.js";
 import "./chunk-OWF2JJKN.js";
 import {
   claimPendingJob,
@@ -32,7 +32,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/worker.ts
 var MAX_CONCURRENT = 2;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.79.2",
+  "version": "0.79.3",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",