mcp-scraper 0.79.2 → 0.79.3

package/dist/{chunk-WEEAXPWK.js → chunk-KNGTQD6U.js}

@@ -8,7 +8,7 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-NGLZ6UJR.js";
+} from "./chunk-5UAFBKJR.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
@@ -24,7 +24,7 @@ import {
   RECAPTCHA_INSTRUCTIONS,
   RequestAbortedError,
   sanitizeVendorName
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/driver/BrowserDriver.ts
 import { chromium } from "playwright-extra";
@@ -2263,17 +2263,22 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
 // src/extractor/PAAExtractor.ts
 var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
 var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
+var PAA_CLICK_MID_CONFIRMATION_MS = 1e3;
 var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
+var PAA_CLICK_DISPATCH_TIMEOUT_MS = 900;
 var PAA_FRONTIER_BATCH_SIZE = 3;
+var MAX_RETAINED_INTERACTION_OBSERVATIONS = 250;
 var PAAExtractor = class {
-  constructor(driver, reporter, onPaaProgress) {
+  constructor(driver, reporter, onPaaProgress, onProviderSession) {
     this.driver = driver;
     this.reporter = reporter;
     this.onPaaProgress = onPaaProgress;
+    this.onProviderSession = onProviderSession;
   }
   driver;
   reporter;
   onPaaProgress;
+  onProviderSession;
   completeness = {
     paaWithoutAnswer: 0,
     paaWithoutSource: 0,
@@ -2288,6 +2293,7 @@ var PAAExtractor = class {
   clickedQuestions = /* @__PURE__ */ new Set();
   paaLifecycle = this.emptyPaaLifecycle(0);
   checkpointMaterial = null;
+  interactionObservations = [];
   emptyPaaLifecycle(requestedQuestions) {
     return {
       requestedQuestions,
@@ -2300,7 +2306,12 @@ var PAAExtractor = class {
       knownQuestions: 0,
       processedQuestions: 0,
       failedInteractions: 0,
-      noGrowthConfirmations: 0
+      noGrowthConfirmations: 0,
+      visibleControls: 0,
+      eligibleControls: 0,
+      dispatchedInteractions: 0,
+      confirmedInteractions: 0,
+      unconfirmedInteractions: 0
     };
   }
   emptyLinkDiagnostics() {
@@ -2562,7 +2573,11 @@ var PAAExtractor = class {
       phase,
       records: this.collectedRows.map((row) => ({ ...row })),
       lifecycle: { ...this.paaLifecycle },
-      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
+      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {},
+      interactions: this.interactionObservations.map((observation) => ({
+        ...observation,
+        confirmationSignals: [...observation.confirmationSignals]
+      }))
     };
     try {
       await this.onPaaProgress(snapshot);
@@ -2643,6 +2658,7 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.interactionObservations = [];
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickAttempts = /* @__PURE__ */ new Map();
     const processedQuestions = /* @__PURE__ */ new Set();
@@ -2650,6 +2666,7 @@ var PAAExtractor = class {
     const capturedItems = /* @__PURE__ */ new Map();
     const MAX_CLICK_ATTEMPTS = 2;
     const EXHAUSTION_CONFIRMATIONS = 3;
+    let batchNumber = 0;
     const itemArgs = {
       sel: PAASelectors.item,
       dataQ: PAASelectors.itemDataQ,
@@ -2659,12 +2676,30 @@ var PAAExtractor = class {
       expandedClass: PAASelectors.expandedClass,
       clickTarget: PAASelectors.clickTarget,
       answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
-      questionsOnly: options.questionsOnly
+      questionsOnly: options.questionsOnly,
+      sourceSel: "a[href]"
     };
     const readItemStates = () => page.evaluate(
       ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
+        const cleanVisibleMaterial = (element) => {
+          if (!element) return "";
+          const parts = [];
+          for (const node of element.childNodes) {
+            if (node.nodeType === Node.TEXT_NODE) {
+              const text2 = node.textContent?.trim();
+              if (text2) parts.push(text2);
+              continue;
+            }
+            if (node.nodeType !== Node.ELEMENT_NODE) continue;
+            const child = node;
+            if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+            const text = cleanVisibleMaterial(child);
+            if (text) parts.push(text);
+          }
+          return parts.join(" ").replace(/\s+/g, " ").trim();
+        };
         return Array.from(document.querySelectorAll(sel)).map((el, index) => {
-          const answerText = (el.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          const answerText = cleanVisibleMaterial(el.querySelector(answerSel)).replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
           const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
           const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
           return {
@@ -2677,34 +2712,133 @@ var PAAExtractor = class {
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
+    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded, controlAttempt, batchNumber2) => {
       const clickStartedMs = Date.now();
+      const baseObservation = {
+        id: `${batchNumber2}:${questionIdFor(q)}:${controlAttempt}`,
+        questionId: questionIdFor(q),
+        question: q,
+        batchNumber: batchNumber2,
+        controlAttempt,
+        observedAt: new Date(clickStartedMs).toISOString(),
+        beforeQuestionCount: beforeCount,
+        expandedBefore: wasExpanded
+      };
       const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      let dispatchErrorCode = null;
       const clicked = clickTarget ? await (async () => {
         if (wasExpanded) {
-          await clickTarget.click({ timeout: 2e3, force: true });
-          await page.waitForTimeout(250);
+          await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
+          await page.waitForTimeout(120);
         }
-        await clickTarget.click({ timeout: 2e3, force: true });
+        await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
         return true;
-      })().catch(() => false) : false;
-      if (!clicked) return { dispatched: false, confirmed: false };
-      const remainingConfirmationMs = Math.max(100, PAA_CLICK_ATTEMPT_BUDGET_MS - (Date.now() - clickStartedMs));
-      const confirmed = await page.waitForFunction(
-        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, questionsOnly, target, min, earliestConfirmationAt }) => {
-          if (Date.now() < earliestConfirmationAt) return false;
-          const items = Array.from(document.querySelectorAll(sel));
-          const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
-          if (!item) return false;
-          const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
-          if (questionsOnly) return expanded || items.length > min;
-          const answerText = (item.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
-          return answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
-        },
-        { ...itemArgs, target: q, min: beforeCount, earliestConfirmationAt: clickStartedMs + PAA_CLICK_EARLIEST_CONFIRMATION_MS },
-        { timeout: remainingConfirmationMs, polling: 100 }
-      ).catch(() => null);
-      return { dispatched: true, confirmed: Boolean(confirmed) };
+      })().catch((error) => {
+        dispatchErrorCode = error instanceof Error && (error.name === "TimeoutError" || /timeout/i.test(error.message)) ? "click_ack_timeout" : "click_failed";
+        return false;
+      }) : false;
+      const dispatchDurationMs = Date.now() - clickStartedMs;
+      if (!clicked && dispatchErrorCode !== "click_ack_timeout") {
+        const observation2 = {
+          ...baseObservation,
+          dispatched: false,
+          outcome: "dispatch_failed",
+          confirmationSignals: [],
+          confirmationCheckpointMs: null,
+          confirmedAtMs: null,
+          dispatchDurationMs,
+          totalDurationMs: dispatchDurationMs,
+          afterQuestionCount: beforeCount,
+          newQuestionCount: 0,
+          expandedAfter: wasExpanded,
+          answerObserved: false,
+          sourceCount: 0,
+          errorCode: itemIndex < 0 ? "control_missing" : dispatchErrorCode ?? "click_failed"
+        };
+        return { dispatched: false, confirmed: false, observation: observation2 };
+      }
+      let latest = {
+        exists: true,
+        expanded: wasExpanded,
+        answerReady: false,
+        sourceCount: 0,
+        questionCount: beforeCount
+      };
+      let signals = [];
+      let confirmationCheckpointMs = null;
+      let confirmedAtMs = null;
+      let confirmed = false;
+      for (const checkpointMs of [PAA_CLICK_EARLIEST_CONFIRMATION_MS, PAA_CLICK_MID_CONFIRMATION_MS, PAA_CLICK_ATTEMPT_BUDGET_MS]) {
+        const remainingMs = clickStartedMs + checkpointMs - Date.now();
+        if (remainingMs > 0) await page.waitForTimeout(remainingMs);
+        latest = await page.evaluate(
+          ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, sourceSel, target }) => {
+            const cleanVisibleMaterial = (element) => {
+              if (!element) return "";
+              const parts = [];
+              for (const node of element.childNodes) {
+                if (node.nodeType === Node.TEXT_NODE) {
+                  const text2 = node.textContent?.trim();
+                  if (text2) parts.push(text2);
+                  continue;
+                }
+                if (node.nodeType !== Node.ELEMENT_NODE) continue;
+                const child = node;
+                if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+                const text = cleanVisibleMaterial(child);
+                if (text) parts.push(text);
+              }
+              return parts.join(" ").replace(/\s+/g, " ").trim();
+            };
+            const items = Array.from(document.querySelectorAll(sel));
+            const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+            if (!item) return { exists: false, expanded: false, answerReady: false, sourceCount: 0, questionCount: items.length };
+            const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
+            const answerElement = item.querySelector(answerSel);
+            const answerText = cleanVisibleMaterial(answerElement);
+            const confirmationText = answerText.replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
+            const answerReady = confirmationText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(confirmationText);
+            return {
+              exists: true,
+              expanded,
+              answerReady,
+              sourceCount: answerReady ? answerElement?.querySelectorAll(sourceSel).length ?? 0 : 0,
+              questionCount: items.length
+            };
+          },
+          { ...itemArgs, target: q }
+        ).catch(() => latest);
+        signals = [
+          ...latest.expanded ? ["expanded"] : [],
+          ...latest.answerReady ? ["answer"] : [],
+          ...latest.sourceCount > 0 ? ["source"] : [],
+          ...latest.questionCount > beforeCount ? ["question_growth"] : []
+        ];
+        if (latest.answerReady || latest.sourceCount > 0 || latest.questionCount > beforeCount) {
+          confirmed = true;
+          confirmationCheckpointMs = checkpointMs;
+          confirmedAtMs = Date.now() - clickStartedMs;
+          break;
+        }
+      }
+      const totalDurationMs = Date.now() - clickStartedMs;
+      const observation = {
+        ...baseObservation,
+        dispatched: clicked,
+        outcome: confirmed ? "confirmed" : clicked ? "unconfirmed" : "dispatch_failed",
+        confirmationSignals: signals,
+        confirmationCheckpointMs,
+        confirmedAtMs,
+        dispatchDurationMs,
+        totalDurationMs,
+        afterQuestionCount: latest.questionCount,
+        newQuestionCount: Math.max(0, latest.questionCount - beforeCount),
+        expandedAfter: latest.expanded,
+        answerObserved: latest.answerReady,
+        sourceCount: latest.sourceCount,
+        errorCode: !clicked ? dispatchErrorCode : confirmed ? null : latest.exists ? "confirmation_timeout" : "control_missing"
+      };
+      return { dispatched: clicked, confirmed, observation };
     };
     let round = 0;
     expansion: while (true) {
@@ -2735,7 +2869,8 @@ var PAAExtractor = class {
         targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
         knownQuestions: orderedQs.length,
         processedQuestions: processedQuestions.size,
-        failedInteractions: failedInteractions.size
+        failedInteractions: failedInteractions.size,
+        visibleControls: states.length
       };
       if (discovered) await this.emitProgress("discovered");
       if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
@@ -2783,6 +2918,7 @@ var PAAExtractor = class {
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
       const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
+      this.paaLifecycle = { ...this.paaLifecycle, eligibleControls: clickable.length };
       if (targets.length === 0) {
         if (seenQs.size >= options.maxQuestions) {
           this.paaLifecycle = {
@@ -2832,6 +2968,7 @@ var PAAExtractor = class {
         break;
       }
       await this.throwIfCaptcha(page, "Google PAA expansion");
+      batchNumber += 1;
       for (const target of targets) {
         this.reporter.onDepth(++round);
         if ((clickAttempts.get(target.q) ?? 0) > 0) {
@@ -2844,10 +2981,21 @@ var PAAExtractor = class {
       }
       const expansionResults = await Promise.all(targets.map(async (target) => ({
         target,
-        result: await expandOneItemSerially(target.q, states.length, target.index, target.expanded)
+        result: await expandOneItemSerially(
+          target.q,
+          states.length,
+          target.index,
+          target.expanded,
+          clickAttempts.get(target.q) ?? 1,
+          batchNumber
+        )
       })));
       for (const { target, result } of expansionResults) {
-        if (result.dispatched) this.clickedQuestions.add(target.q);
+        if (result.dispatched || result.confirmed) this.clickedQuestions.add(target.q);
+        this.interactionObservations.push(result.observation);
+      }
+      if (this.interactionObservations.length > MAX_RETAINED_INTERACTION_OBSERVATIONS) {
+        this.interactionObservations.splice(0, this.interactionObservations.length - MAX_RETAINED_INTERACTION_OBSERVATIONS);
       }
       if (!options.questionsOnly) {
         const targetQuestions = new Set(targets.map((target) => target.q));
@@ -2873,7 +3021,10 @@ var PAAExtractor = class {
         ...this.paaLifecycle,
         processedQuestions: processedQuestions.size,
         failedInteractions: failedInteractions.size,
-        noGrowthConfirmations: 0
+        noGrowthConfirmations: 0,
+        dispatchedInteractions: this.interactionObservations.filter((value) => value.dispatched).length,
+        confirmedInteractions: this.interactionObservations.filter((value) => value.outcome === "confirmed").length,
+        unconfirmedInteractions: this.interactionObservations.filter((value) => value.outcome !== "confirmed").length
       };
       await this.emitProgress("click_observed");
     }
@@ -3467,6 +3618,18 @@ var PAAExtractor = class {
     try {
       this.throwIfAborted(signal);
       await this.driver.launch(config);
+      const providerSession = this.driver.getProviderSessionMetadata?.();
+      if (providerSession?.providerSessionId && this.onProviderSession) {
+        try {
+          await this.onProviderSession(providerSession);
+        } catch (err) {
+          console.warn(JSON.stringify({
+            event: "paa_provider_session_sink_failed",
+            provider: providerSession.provider,
+            message: err instanceof Error ? err.message : String(err)
+          }));
+        }
+      }
       this.throwIfAborted(signal);
       const locationResolution = options.location ? resolveLocation(options.location) : null;
       const canonicalLocation = options.location ? locationResolution?.canonical ?? options.location : null;
@@ -4602,10 +4765,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
     }));
   }
 }
-async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
+async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress, onProviderSession) {
   const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
   const reporter = new ProgressReporter();
-  const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
+  const extractor = new PAAExtractor(driver, reporter, onPaaProgress, onProviderSession);
   if (signal?.aborted) {
     return {
       result: null,
@@ -4750,12 +4913,20 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
+      const onProviderSession = (metadata) => emitAttemptEvent(onAttemptEvent, {
+        type: "provider_session_started",
+        attemptNumber,
+        maxAttempts,
+        provider: metadata.provider,
+        providerSessionId: metadata.providerSessionId,
+        observedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
       const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
       const attempt = baseCtx ? await runWithCostContext({
         ...baseCtx,
         forceHeadless: allowHeadlessTest,
         forceHeadful: !allowHeadlessTest
-      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
+      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);

package/dist/{chunk-6FL7ZZCF.js → chunk-MVB25X55.js}

@@ -1,6 +1,6 @@
 // release-message.json
 var release_message_default = {
-  message: "PAA harvests now reject Google error placeholders, retry interrupted controls without claiming the frontier ended, advance the newest question frontier in bounded batches, preserve complete answers and sources, and resolve retained Google redirect evidence even after the browser session closes."
+  message: "PAA harvests now distinguish click acknowledgement from confirmed Google DOM effects, preserve every observed interaction at 0.7, 1.0, and 1.4-second checkpoints, reject hidden or scripted placeholder text as answers, correlate attempts to Bright Data sessions immediately, and recover interrupted work without claiming the question frontier ended."
 };
 
 // src/install-terminal.ts

package/dist/{chunk-47ZCUBM5.js → chunk-NVUKO5NN.js}

@@ -6,11 +6,12 @@ import {
   LocationMismatchError,
   PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
   RequestAbortedError,
+  attachHarvestAttemptProviderSession,
   buildPublicErrorEnvelope,
   finishHarvestAttempt,
   publicErrorMessage,
   startHarvestAttempt
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -186,6 +187,16 @@ function createHarvestAttemptRecorder(jobId, userId) {
       });
       return;
     }
+    if (event.type === "provider_session_started") {
+      await attachHarvestAttemptProviderSession({
+        jobId,
+        attemptNumber: event.attemptNumber,
+        browserProvider: event.provider,
+        providerSessionId: event.providerSessionId,
+        observedAt: event.observedAt
+      });
+      return;
+    }
     await finishHarvestAttempt({
       jobId,
       attemptNumber: event.attemptNumber,

package/dist/{chunk-XUM2LEGU.js → chunk-TCYZ7HMU.js}

@@ -12,13 +12,13 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "./chunk-JECPQBLZ.js";
+} from "./chunk-7UZAVM4J.js";
 import {
   renderInstallTerminal
-} from "./chunk-6FL7ZZCF.js";
+} from "./chunk-MVB25X55.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2RTELL6M.js";
+} from "./chunk-IHYG52ZW.js";
 
 // src/mcp/stdio-runtime.ts
 import { readFileSync } from "fs";

package/dist/{chunk-SL7YVA4S.js → chunk-YBR32LOD.js}

@@ -15,7 +15,7 @@ import {
   sanitizeVendorName,
   serializePublicErrorEnvelope,
   settleDebitMcIdempotent
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/site-extract-content-store.ts
 import { createHash } from "crypto";

package/dist/{chunk-OU4FV2RP.js → chunk-YXNDOQXN.js}

@@ -488,6 +488,7 @@ var CORE_SCHEMA_COLUMNS = {
     "debug_json",
     "browser_provider",
     "provider_session_id",
+    "provider_session_observed_at",
     "provider_lookup_status",
     "provider_status",
     "provider_error_code",
@@ -711,6 +712,7 @@ async function migrate(options = {}) {
       debug_json               TEXT,
       browser_provider         TEXT,
       provider_session_id      TEXT,
+      provider_session_observed_at TEXT,
       provider_lookup_status   TEXT,
       provider_status          TEXT,
       provider_error_code      TEXT,
@@ -737,6 +739,7 @@ async function migrate(options = {}) {
   for (const column of [
     "browser_provider TEXT",
     "provider_session_id TEXT",
+    "provider_session_observed_at TEXT",
     "provider_lookup_status TEXT",
     "provider_status TEXT",
     "provider_error_code TEXT",
@@ -1912,6 +1915,7 @@ function rowToHarvestAttemptLog(row) {
     browser_close_error: row.browser_close_error != null ? String(row.browser_close_error) : null,
     browser_provider: row.browser_provider != null ? String(row.browser_provider) : null,
     provider_session_id: row.provider_session_id != null ? String(row.provider_session_id) : null,
+    provider_session_observed_at: row.provider_session_observed_at != null ? String(row.provider_session_observed_at) : null,
     provider_lookup_status: row.provider_lookup_status != null ? String(row.provider_lookup_status) : null,
     provider_status: row.provider_status != null ? String(row.provider_status) : null,
     provider_error_code: row.provider_error_code != null ? String(row.provider_error_code) : null,
@@ -1991,6 +1995,49 @@ async function startHarvestAttempt(input) {
     ]
   });
 }
+async function attachHarvestAttemptProviderSession(input) {
+  const result = await getDb().execute({
+    sql: `
+      UPDATE harvest_attempts SET
+        browser_provider = ?,
+        provider_session_id = ?,
+        provider_session_observed_at = COALESCE(provider_session_observed_at, ?),
+        provider_lookup_status = CASE
+          WHEN provider_lookup_status = 'complete' THEN provider_lookup_status
+          ELSE 'pending'
+        END
+      WHERE job_id = ? AND attempt_number = ? AND status = 'running'
+    `,
+    args: [
+      input.browserProvider,
+      input.providerSessionId,
+      input.observedAt,
+      input.jobId,
+      input.attemptNumber
+    ]
+  });
+  return result.rowsAffected === 1;
+}
+async function finalizeExpiredHarvestAttempts(jobId, willRetry, completedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const result = await getDb().execute({
+    sql: `
+      UPDATE harvest_attempts SET
+        status = 'finished',
+        outcome = COALESCE(outcome, 'timeout'),
+        question_count = COALESCE(question_count, 0),
+        duration_ms = COALESCE(duration_ms, MAX(0, CAST((julianday(?) - julianday(created_at)) * 86400000 AS INTEGER))),
+        error = COALESCE(error, 'Durable worker lease expired before attempt completion'),
+        will_retry = ?,
+        provider_error_code = COALESCE(provider_error_code, 'timeout'),
+        provider_error_message = COALESCE(provider_error_message, 'Durable worker lease expired before attempt completion'),
+        provider_reason_source = COALESCE(provider_reason_source, 'client_runtime'),
+        completed_at = COALESCE(completed_at, ?)
+      WHERE job_id = ? AND status = 'running'
+    `,
+    args: [completedAt, willRetry ? 1 : 0, completedAt, jobId]
+  });
+  return result.rowsAffected;
+}
 async function finishHarvestAttempt(input) {
   await getDb().execute({
     sql: `
@@ -3869,6 +3916,8 @@ export {
   createRunningJob,
   getOrCreateRunningJob,
   startHarvestAttempt,
+  attachHarvestAttemptProviderSession,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   updateHarvestAttemptProviderTelemetry,
   listHarvestAttemptsPendingProviderTelemetry,

package/dist/{db-TA567JLA.js → db-Z34LPZNR.js}

@@ -7,6 +7,7 @@ import {
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
   appendWorkflowArtifacts,
+  attachHarvestAttemptProviderSession,
   cancelJob,
   checkRateLimit,
   checkpointDurablePaaJob,
@@ -49,6 +50,7 @@ import {
   failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   generateApiKey,
   getBillingLedgerSummary,
@@ -137,7 +139,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,
@@ -147,6 +149,7 @@ export {
   SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep,
   appendWorkflowArtifacts,
+  attachHarvestAttemptProviderSession,
   cancelJob,
   checkRateLimit,
   checkpointDurablePaaJob,
@@ -189,6 +192,7 @@ export {
   failRunningJob,
   failStripeEvent,
   failWorkflowRunRecord,
+  finalizeExpiredHarvestAttempts,
   finishHarvestAttempt,
   generateApiKey,
   getBillingLedgerSummary,

package/dist/{extract-bundle-I2GNC25L.js → extract-bundle-S3ZGNFML.js}

@@ -4,13 +4,13 @@ import {
   downloadAsset,
   embedCommonsTexts,
   publicizeExtractionFailure
-} from "./chunk-BTQ23KCT.js";
+} from "./chunk-F5GQJWZU.js";
 import {
   createSiteExtractBundleArtifactStream,
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-SL7YVA4S.js";
+} from "./chunk-YBR32LOD.js";
 import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
@@ -23,7 +23,7 @@ import "./chunk-VFCUGTCF.js";
 import "./chunk-GGZEC22A.js";
 import {
   getDb
-} from "./chunk-OU4FV2RP.js";
+} from "./chunk-YXNDOQXN.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";

package/dist/{gmail-service-GGQVJUC3.js → gmail-service-YFS2OV4S.js}

@@ -7,12 +7,12 @@ import {
   GmailServiceError,
   normalizeGmailMessage,
   parseGmailAddresses
-} from "./chunk-LB2GNACG.js";
-import "./chunk-Q5L2OCOQ.js";
+} from "./chunk-3423R23T.js";
+import "./chunk-C5Z4OFKW.js";
 import "./chunk-T3MZISOF.js";
 import "./chunk-VFCUGTCF.js";
-import "./chunk-NGLZ6UJR.js";
-import "./chunk-OU4FV2RP.js";
+import "./chunk-5UAFBKJR.js";
+import "./chunk-YXNDOQXN.js";
 export {
   GMAIL_ATTACHMENT_REF_TTL_MS,
   GMAIL_INLINE_BODY_BYTES,