mcp-scraper 0.79.2 → 0.79.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +4 -0
  2. package/README.md +1 -1
  3. package/dist/{analytics-repository-VEUDEPWI.js → analytics-repository-YOY5KMOD.js} +3 -3
  4. package/dist/bin/api-server.js +4 -4
  5. package/dist/bin/mcp-scraper-cli.js +1 -1
  6. package/dist/bin/mcp-scraper-core.js +7 -7
  7. package/dist/bin/mcp-scraper-install.js +2 -2
  8. package/dist/bin/mcp-stdio-server.js +7 -7
  9. package/dist/bin/paa-harvest.js +3 -3
  10. package/dist/{chunk-LB2GNACG.js → chunk-3423R23T.js} +3 -3
  11. package/dist/{chunk-NGLZ6UJR.js → chunk-5UAFBKJR.js} +1 -1
  12. package/dist/{chunk-B6L73AO4.js → chunk-6HAV7LCE.js} +3 -1
  13. package/dist/{chunk-JECPQBLZ.js → chunk-7UZAVM4J.js} +138 -11
  14. package/dist/{chunk-Q5L2OCOQ.js → chunk-C5Z4OFKW.js} +1 -1
  15. package/dist/{chunk-KAGHJQFO.js → chunk-EQGTEHLZ.js} +1 -1
  16. package/dist/{chunk-BTQ23KCT.js → chunk-F5GQJWZU.js} +1 -1
  17. package/dist/{chunk-DUWZQSSJ.js → chunk-HEUMDX4I.js} +1 -1
  18. package/dist/{chunk-2RTELL6M.js → chunk-IHYG52ZW.js} +1 -1
  19. package/dist/{chunk-OUBH6XMY.js → chunk-K3Z5AQYE.js} +1 -1
  20. package/dist/{chunk-WEEAXPWK.js → chunk-KNGTQD6U.js} +207 -36
  21. package/dist/{chunk-6FL7ZZCF.js → chunk-MVB25X55.js} +1 -1
  22. package/dist/{chunk-47ZCUBM5.js → chunk-NVUKO5NN.js} +12 -1
  23. package/dist/{chunk-XUM2LEGU.js → chunk-TCYZ7HMU.js} +3 -3
  24. package/dist/{chunk-SL7YVA4S.js → chunk-YBR32LOD.js} +1 -1
  25. package/dist/{chunk-OU4FV2RP.js → chunk-YXNDOQXN.js} +49 -0
  26. package/dist/{db-TA567JLA.js → db-Z34LPZNR.js} +5 -1
  27. package/dist/{extract-bundle-I2GNC25L.js → extract-bundle-S3ZGNFML.js} +3 -3
  28. package/dist/{gmail-service-GGQVJUC3.js → gmail-service-YFS2OV4S.js} +4 -4
  29. package/dist/index.cjs +205 -34
  30. package/dist/index.d.cts +35 -0
  31. package/dist/index.d.ts +35 -0
  32. package/dist/index.js +3 -3
  33. package/dist/{lead-list-enrichment-repository-5LEWW4X7.js → lead-list-enrichment-repository-36RPVV6N.js} +2 -2
  34. package/dist/{location-data-repository-AHKDXTEQ.js → location-data-repository-WPRG62GE.js} +2 -2
  35. package/dist/{server-5OSHEEVT.js → server-MDWFOUTS.js} +73 -28
  36. package/dist/{site-extract-repository-KF736IR6.js → site-extract-repository-WAJMIDPJ.js} +2 -2
  37. package/dist/{worker-2TYCYFPM.js → worker-3LUFQWWJ.js} +5 -5
  38. package/package.json +1 -1
package/dist/index.cjs CHANGED
@@ -2807,17 +2807,22 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
2807
2807
  // src/extractor/PAAExtractor.ts
2808
2808
  var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
2809
2809
  var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
2810
+ var PAA_CLICK_MID_CONFIRMATION_MS = 1e3;
2810
2811
  var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
2812
+ var PAA_CLICK_DISPATCH_TIMEOUT_MS = 900;
2811
2813
  var PAA_FRONTIER_BATCH_SIZE = 3;
2814
+ var MAX_RETAINED_INTERACTION_OBSERVATIONS = 250;
2812
2815
  var PAAExtractor = class {
2813
- constructor(driver, reporter, onPaaProgress) {
2816
+ constructor(driver, reporter, onPaaProgress, onProviderSession) {
2814
2817
  this.driver = driver;
2815
2818
  this.reporter = reporter;
2816
2819
  this.onPaaProgress = onPaaProgress;
2820
+ this.onProviderSession = onProviderSession;
2817
2821
  }
2818
2822
  driver;
2819
2823
  reporter;
2820
2824
  onPaaProgress;
2825
+ onProviderSession;
2821
2826
  completeness = {
2822
2827
  paaWithoutAnswer: 0,
2823
2828
  paaWithoutSource: 0,
@@ -2832,6 +2837,7 @@ var PAAExtractor = class {
2832
2837
  clickedQuestions = /* @__PURE__ */ new Set();
2833
2838
  paaLifecycle = this.emptyPaaLifecycle(0);
2834
2839
  checkpointMaterial = null;
2840
+ interactionObservations = [];
2835
2841
  emptyPaaLifecycle(requestedQuestions) {
2836
2842
  return {
2837
2843
  requestedQuestions,
@@ -2844,7 +2850,12 @@ var PAAExtractor = class {
2844
2850
  knownQuestions: 0,
2845
2851
  processedQuestions: 0,
2846
2852
  failedInteractions: 0,
2847
- noGrowthConfirmations: 0
2853
+ noGrowthConfirmations: 0,
2854
+ visibleControls: 0,
2855
+ eligibleControls: 0,
2856
+ dispatchedInteractions: 0,
2857
+ confirmedInteractions: 0,
2858
+ unconfirmedInteractions: 0
2848
2859
  };
2849
2860
  }
2850
2861
  emptyLinkDiagnostics() {
@@ -3106,7 +3117,11 @@ var PAAExtractor = class {
3106
3117
  phase,
3107
3118
  records: this.collectedRows.map((row) => ({ ...row })),
3108
3119
  lifecycle: { ...this.paaLifecycle },
3109
- ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
3120
+ ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {},
3121
+ interactions: this.interactionObservations.map((observation) => ({
3122
+ ...observation,
3123
+ confirmationSignals: [...observation.confirmationSignals]
3124
+ }))
3110
3125
  };
3111
3126
  try {
3112
3127
  await this.onPaaProgress(snapshot);
@@ -3187,6 +3202,7 @@ var PAAExtractor = class {
3187
3202
  const seenQs = /* @__PURE__ */ new Set();
3188
3203
  const orderedQs = [];
3189
3204
  this.clickedQuestions = /* @__PURE__ */ new Set();
3205
+ this.interactionObservations = [];
3190
3206
  const abandonedNeverReclick = /* @__PURE__ */ new Set();
3191
3207
  const clickAttempts = /* @__PURE__ */ new Map();
3192
3208
  const processedQuestions = /* @__PURE__ */ new Set();
@@ -3194,6 +3210,7 @@ var PAAExtractor = class {
3194
3210
  const capturedItems = /* @__PURE__ */ new Map();
3195
3211
  const MAX_CLICK_ATTEMPTS = 2;
3196
3212
  const EXHAUSTION_CONFIRMATIONS = 3;
3213
+ let batchNumber = 0;
3197
3214
  const itemArgs = {
3198
3215
  sel: PAASelectors.item,
3199
3216
  dataQ: PAASelectors.itemDataQ,
@@ -3203,12 +3220,30 @@ var PAAExtractor = class {
3203
3220
  expandedClass: PAASelectors.expandedClass,
3204
3221
  clickTarget: PAASelectors.clickTarget,
3205
3222
  answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
3206
- questionsOnly: options.questionsOnly
3223
+ questionsOnly: options.questionsOnly,
3224
+ sourceSel: "a[href]"
3207
3225
  };
3208
3226
  const readItemStates = () => page.evaluate(
3209
3227
  ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
3228
+ const cleanVisibleMaterial = (element) => {
3229
+ if (!element) return "";
3230
+ const parts = [];
3231
+ for (const node of element.childNodes) {
3232
+ if (node.nodeType === Node.TEXT_NODE) {
3233
+ const text2 = node.textContent?.trim();
3234
+ if (text2) parts.push(text2);
3235
+ continue;
3236
+ }
3237
+ if (node.nodeType !== Node.ELEMENT_NODE) continue;
3238
+ const child = node;
3239
+ if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
3240
+ const text = cleanVisibleMaterial(child);
3241
+ if (text) parts.push(text);
3242
+ }
3243
+ return parts.join(" ").replace(/\s+/g, " ").trim();
3244
+ };
3210
3245
  return Array.from(document.querySelectorAll(sel)).map((el, index) => {
3211
- const answerText = (el.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
3246
+ const answerText = cleanVisibleMaterial(el.querySelector(answerSel)).replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
3212
3247
  const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
3213
3248
  const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
3214
3249
  return {
@@ -3221,34 +3256,133 @@ var PAAExtractor = class {
3221
3256
  },
3222
3257
  itemArgs
3223
3258
  );
3224
- const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
3259
+ const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded, controlAttempt, batchNumber2) => {
3225
3260
  const clickStartedMs = Date.now();
3261
+ const baseObservation = {
3262
+ id: `${batchNumber2}:${questionIdFor(q)}:${controlAttempt}`,
3263
+ questionId: questionIdFor(q),
3264
+ question: q,
3265
+ batchNumber: batchNumber2,
3266
+ controlAttempt,
3267
+ observedAt: new Date(clickStartedMs).toISOString(),
3268
+ beforeQuestionCount: beforeCount,
3269
+ expandedBefore: wasExpanded
3270
+ };
3226
3271
  const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
3272
+ let dispatchErrorCode = null;
3227
3273
  const clicked = clickTarget ? await (async () => {
3228
3274
  if (wasExpanded) {
3229
- await clickTarget.click({ timeout: 2e3, force: true });
3230
- await page.waitForTimeout(250);
3275
+ await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
3276
+ await page.waitForTimeout(120);
3231
3277
  }
3232
- await clickTarget.click({ timeout: 2e3, force: true });
3278
+ await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
3233
3279
  return true;
3234
- })().catch(() => false) : false;
3235
- if (!clicked) return { dispatched: false, confirmed: false };
3236
- const remainingConfirmationMs = Math.max(100, PAA_CLICK_ATTEMPT_BUDGET_MS - (Date.now() - clickStartedMs));
3237
- const confirmed = await page.waitForFunction(
3238
- ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, questionsOnly, target, min, earliestConfirmationAt }) => {
3239
- if (Date.now() < earliestConfirmationAt) return false;
3240
- const items = Array.from(document.querySelectorAll(sel));
3241
- const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
3242
- if (!item) return false;
3243
- const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
3244
- if (questionsOnly) return expanded || items.length > min;
3245
- const answerText = (item.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
3246
- return answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
3247
- },
3248
- { ...itemArgs, target: q, min: beforeCount, earliestConfirmationAt: clickStartedMs + PAA_CLICK_EARLIEST_CONFIRMATION_MS },
3249
- { timeout: remainingConfirmationMs, polling: 100 }
3250
- ).catch(() => null);
3251
- return { dispatched: true, confirmed: Boolean(confirmed) };
3280
+ })().catch((error) => {
3281
+ dispatchErrorCode = error instanceof Error && (error.name === "TimeoutError" || /timeout/i.test(error.message)) ? "click_ack_timeout" : "click_failed";
3282
+ return false;
3283
+ }) : false;
3284
+ const dispatchDurationMs = Date.now() - clickStartedMs;
3285
+ if (!clicked && dispatchErrorCode !== "click_ack_timeout") {
3286
+ const observation2 = {
3287
+ ...baseObservation,
3288
+ dispatched: false,
3289
+ outcome: "dispatch_failed",
3290
+ confirmationSignals: [],
3291
+ confirmationCheckpointMs: null,
3292
+ confirmedAtMs: null,
3293
+ dispatchDurationMs,
3294
+ totalDurationMs: dispatchDurationMs,
3295
+ afterQuestionCount: beforeCount,
3296
+ newQuestionCount: 0,
3297
+ expandedAfter: wasExpanded,
3298
+ answerObserved: false,
3299
+ sourceCount: 0,
3300
+ errorCode: itemIndex < 0 ? "control_missing" : dispatchErrorCode ?? "click_failed"
3301
+ };
3302
+ return { dispatched: false, confirmed: false, observation: observation2 };
3303
+ }
3304
+ let latest = {
3305
+ exists: true,
3306
+ expanded: wasExpanded,
3307
+ answerReady: false,
3308
+ sourceCount: 0,
3309
+ questionCount: beforeCount
3310
+ };
3311
+ let signals = [];
3312
+ let confirmationCheckpointMs = null;
3313
+ let confirmedAtMs = null;
3314
+ let confirmed = false;
3315
+ for (const checkpointMs of [PAA_CLICK_EARLIEST_CONFIRMATION_MS, PAA_CLICK_MID_CONFIRMATION_MS, PAA_CLICK_ATTEMPT_BUDGET_MS]) {
3316
+ const remainingMs = clickStartedMs + checkpointMs - Date.now();
3317
+ if (remainingMs > 0) await page.waitForTimeout(remainingMs);
3318
+ latest = await page.evaluate(
3319
+ ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, sourceSel, target }) => {
3320
+ const cleanVisibleMaterial = (element) => {
3321
+ if (!element) return "";
3322
+ const parts = [];
3323
+ for (const node of element.childNodes) {
3324
+ if (node.nodeType === Node.TEXT_NODE) {
3325
+ const text2 = node.textContent?.trim();
3326
+ if (text2) parts.push(text2);
3327
+ continue;
3328
+ }
3329
+ if (node.nodeType !== Node.ELEMENT_NODE) continue;
3330
+ const child = node;
3331
+ if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
3332
+ const text = cleanVisibleMaterial(child);
3333
+ if (text) parts.push(text);
3334
+ }
3335
+ return parts.join(" ").replace(/\s+/g, " ").trim();
3336
+ };
3337
+ const items = Array.from(document.querySelectorAll(sel));
3338
+ const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
3339
+ if (!item) return { exists: false, expanded: false, answerReady: false, sourceCount: 0, questionCount: items.length };
3340
+ const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
3341
+ const answerElement = item.querySelector(answerSel);
3342
+ const answerText = cleanVisibleMaterial(answerElement);
3343
+ const confirmationText = answerText.replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
3344
+ const answerReady = confirmationText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(confirmationText);
3345
+ return {
3346
+ exists: true,
3347
+ expanded,
3348
+ answerReady,
3349
+ sourceCount: answerReady ? answerElement?.querySelectorAll(sourceSel).length ?? 0 : 0,
3350
+ questionCount: items.length
3351
+ };
3352
+ },
3353
+ { ...itemArgs, target: q }
3354
+ ).catch(() => latest);
3355
+ signals = [
3356
+ ...latest.expanded ? ["expanded"] : [],
3357
+ ...latest.answerReady ? ["answer"] : [],
3358
+ ...latest.sourceCount > 0 ? ["source"] : [],
3359
+ ...latest.questionCount > beforeCount ? ["question_growth"] : []
3360
+ ];
3361
+ if (latest.answerReady || latest.sourceCount > 0 || latest.questionCount > beforeCount) {
3362
+ confirmed = true;
3363
+ confirmationCheckpointMs = checkpointMs;
3364
+ confirmedAtMs = Date.now() - clickStartedMs;
3365
+ break;
3366
+ }
3367
+ }
3368
+ const totalDurationMs = Date.now() - clickStartedMs;
3369
+ const observation = {
3370
+ ...baseObservation,
3371
+ dispatched: clicked,
3372
+ outcome: confirmed ? "confirmed" : clicked ? "unconfirmed" : "dispatch_failed",
3373
+ confirmationSignals: signals,
3374
+ confirmationCheckpointMs,
3375
+ confirmedAtMs,
3376
+ dispatchDurationMs,
3377
+ totalDurationMs,
3378
+ afterQuestionCount: latest.questionCount,
3379
+ newQuestionCount: Math.max(0, latest.questionCount - beforeCount),
3380
+ expandedAfter: latest.expanded,
3381
+ answerObserved: latest.answerReady,
3382
+ sourceCount: latest.sourceCount,
3383
+ errorCode: !clicked ? dispatchErrorCode : confirmed ? null : latest.exists ? "confirmation_timeout" : "control_missing"
3384
+ };
3385
+ return { dispatched: clicked, confirmed, observation };
3252
3386
  };
3253
3387
  let round = 0;
3254
3388
  expansion: while (true) {
@@ -3279,7 +3413,8 @@ var PAAExtractor = class {
3279
3413
  targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
3280
3414
  knownQuestions: orderedQs.length,
3281
3415
  processedQuestions: processedQuestions.size,
3282
- failedInteractions: failedInteractions.size
3416
+ failedInteractions: failedInteractions.size,
3417
+ visibleControls: states.length
3283
3418
  };
3284
3419
  if (discovered) await this.emitProgress("discovered");
3285
3420
  if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
@@ -3327,6 +3462,7 @@ var PAAExtractor = class {
3327
3462
  const kept = new Set(orderedQs.slice(0, options.maxQuestions));
3328
3463
  const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
3329
3464
  const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
3465
+ this.paaLifecycle = { ...this.paaLifecycle, eligibleControls: clickable.length };
3330
3466
  if (targets.length === 0) {
3331
3467
  if (seenQs.size >= options.maxQuestions) {
3332
3468
  this.paaLifecycle = {
@@ -3376,6 +3512,7 @@ var PAAExtractor = class {
3376
3512
  break;
3377
3513
  }
3378
3514
  await this.throwIfCaptcha(page, "Google PAA expansion");
3515
+ batchNumber += 1;
3379
3516
  for (const target of targets) {
3380
3517
  this.reporter.onDepth(++round);
3381
3518
  if ((clickAttempts.get(target.q) ?? 0) > 0) {
@@ -3388,10 +3525,21 @@ var PAAExtractor = class {
3388
3525
  }
3389
3526
  const expansionResults = await Promise.all(targets.map(async (target) => ({
3390
3527
  target,
3391
- result: await expandOneItemSerially(target.q, states.length, target.index, target.expanded)
3528
+ result: await expandOneItemSerially(
3529
+ target.q,
3530
+ states.length,
3531
+ target.index,
3532
+ target.expanded,
3533
+ clickAttempts.get(target.q) ?? 1,
3534
+ batchNumber
3535
+ )
3392
3536
  })));
3393
3537
  for (const { target, result } of expansionResults) {
3394
- if (result.dispatched) this.clickedQuestions.add(target.q);
3538
+ if (result.dispatched || result.confirmed) this.clickedQuestions.add(target.q);
3539
+ this.interactionObservations.push(result.observation);
3540
+ }
3541
+ if (this.interactionObservations.length > MAX_RETAINED_INTERACTION_OBSERVATIONS) {
3542
+ this.interactionObservations.splice(0, this.interactionObservations.length - MAX_RETAINED_INTERACTION_OBSERVATIONS);
3395
3543
  }
3396
3544
  if (!options.questionsOnly) {
3397
3545
  const targetQuestions = new Set(targets.map((target) => target.q));
@@ -3417,7 +3565,10 @@ var PAAExtractor = class {
3417
3565
  ...this.paaLifecycle,
3418
3566
  processedQuestions: processedQuestions.size,
3419
3567
  failedInteractions: failedInteractions.size,
3420
- noGrowthConfirmations: 0
3568
+ noGrowthConfirmations: 0,
3569
+ dispatchedInteractions: this.interactionObservations.filter((value) => value.dispatched).length,
3570
+ confirmedInteractions: this.interactionObservations.filter((value) => value.outcome === "confirmed").length,
3571
+ unconfirmedInteractions: this.interactionObservations.filter((value) => value.outcome !== "confirmed").length
3421
3572
  };
3422
3573
  await this.emitProgress("click_observed");
3423
3574
  }
@@ -4011,6 +4162,18 @@ var PAAExtractor = class {
4011
4162
  try {
4012
4163
  this.throwIfAborted(signal);
4013
4164
  await this.driver.launch(config);
4165
+ const providerSession = this.driver.getProviderSessionMetadata?.();
4166
+ if (providerSession?.providerSessionId && this.onProviderSession) {
4167
+ try {
4168
+ await this.onProviderSession(providerSession);
4169
+ } catch (err) {
4170
+ console.warn(JSON.stringify({
4171
+ event: "paa_provider_session_sink_failed",
4172
+ provider: providerSession.provider,
4173
+ message: err instanceof Error ? err.message : String(err)
4174
+ }));
4175
+ }
4176
+ }
4014
4177
  this.throwIfAborted(signal);
4015
4178
  const locationResolution = options.location ? resolveLocation(options.location) : null;
4016
4179
  const canonicalLocation = options.location ? locationResolution?.canonical ?? options.location : null;
@@ -5142,10 +5305,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
5142
5305
  }));
5143
5306
  }
5144
5307
  }
5145
- async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
5308
+ async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress, onProviderSession) {
5146
5309
  const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
5147
5310
  const reporter = new ProgressReporter();
5148
- const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
5311
+ const extractor = new PAAExtractor(driver, reporter, onPaaProgress, onProviderSession);
5149
5312
  if (signal?.aborted) {
5150
5313
  return {
5151
5314
  result: null,
@@ -5290,12 +5453,20 @@ async function harvest(rawOptions) {
5290
5453
  max_questions: attemptOptions.maxQuestions
5291
5454
  }));
5292
5455
  const baseCtx = currentCostContext();
5456
+ const onProviderSession = (metadata) => emitAttemptEvent(onAttemptEvent, {
5457
+ type: "provider_session_started",
5458
+ attemptNumber,
5459
+ maxAttempts,
5460
+ provider: metadata.provider,
5461
+ providerSessionId: metadata.providerSessionId,
5462
+ observedAt: (/* @__PURE__ */ new Date()).toISOString()
5463
+ });
5293
5464
  const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
5294
5465
  const attempt = baseCtx ? await runWithCostContext({
5295
5466
  ...baseCtx,
5296
5467
  forceHeadless: allowHeadlessTest,
5297
5468
  forceHeadful: !allowHeadlessTest
5298
- }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
5469
+ }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession);
5299
5470
  if (attempt.error) {
5300
5471
  const err = attempt.error;
5301
5472
  const outcome2 = classifyAttemptError(err);
package/dist/index.d.cts CHANGED
@@ -141,6 +141,40 @@ interface PaaLifecycleDiagnostics {
141
141
  processedQuestions: number;
142
142
  failedInteractions: number;
143
143
  noGrowthConfirmations: number;
144
+ visibleControls?: number;
145
+ eligibleControls?: number;
146
+ dispatchedInteractions?: number;
147
+ confirmedInteractions?: number;
148
+ unconfirmedInteractions?: number;
149
+ }
150
+ type PaaInteractionOutcome = 'confirmed' | 'unconfirmed' | 'dispatch_failed';
151
+ type PaaInteractionSignal = 'expanded' | 'answer' | 'source' | 'question_growth';
152
+ /** One bounded, auditable interaction with a rendered PAA control. Question
153
+ * growth is recorded separately from control-local answer/expanded signals so
154
+ * a multi-control batch never pretends it can attribute shared DOM growth to a
155
+ * specific click with more certainty than the browser exposed. */
156
+ interface PaaInteractionObservation {
157
+ id: string;
158
+ questionId: string;
159
+ question: string;
160
+ batchNumber: number;
161
+ controlAttempt: number;
162
+ observedAt: string;
163
+ dispatched: boolean;
164
+ outcome: PaaInteractionOutcome;
165
+ confirmationSignals: PaaInteractionSignal[];
166
+ confirmationCheckpointMs: 700 | 1000 | 1400 | null;
167
+ confirmedAtMs: number | null;
168
+ dispatchDurationMs: number;
169
+ totalDurationMs: number;
170
+ beforeQuestionCount: number;
171
+ afterQuestionCount: number;
172
+ newQuestionCount: number;
173
+ expandedBefore: boolean;
174
+ expandedAfter: boolean;
175
+ answerObserved: boolean;
176
+ sourceCount: number;
177
+ errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
144
178
  }
145
179
  type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
146
180
  /** Material already observed before PAA expansion completes. It is checkpointed
@@ -164,6 +198,7 @@ interface PaaProgressSnapshot {
164
198
  records: FlatRow[];
165
199
  lifecycle?: PaaLifecycleDiagnostics;
166
200
  material?: PaaProgressMaterial;
201
+ interactions?: PaaInteractionObservation[];
167
202
  }
168
203
  interface HarvestDiagnostics {
169
204
  completionStatus: HarvestCompletionStatus;
package/dist/index.d.ts CHANGED
@@ -141,6 +141,40 @@ interface PaaLifecycleDiagnostics {
141
141
  processedQuestions: number;
142
142
  failedInteractions: number;
143
143
  noGrowthConfirmations: number;
144
+ visibleControls?: number;
145
+ eligibleControls?: number;
146
+ dispatchedInteractions?: number;
147
+ confirmedInteractions?: number;
148
+ unconfirmedInteractions?: number;
149
+ }
150
+ type PaaInteractionOutcome = 'confirmed' | 'unconfirmed' | 'dispatch_failed';
151
+ type PaaInteractionSignal = 'expanded' | 'answer' | 'source' | 'question_growth';
152
+ /** One bounded, auditable interaction with a rendered PAA control. Question
153
+ * growth is recorded separately from control-local answer/expanded signals so
154
+ * a multi-control batch never pretends it can attribute shared DOM growth to a
155
+ * specific click with more certainty than the browser exposed. */
156
+ interface PaaInteractionObservation {
157
+ id: string;
158
+ questionId: string;
159
+ question: string;
160
+ batchNumber: number;
161
+ controlAttempt: number;
162
+ observedAt: string;
163
+ dispatched: boolean;
164
+ outcome: PaaInteractionOutcome;
165
+ confirmationSignals: PaaInteractionSignal[];
166
+ confirmationCheckpointMs: 700 | 1000 | 1400 | null;
167
+ confirmedAtMs: number | null;
168
+ dispatchDurationMs: number;
169
+ totalDurationMs: number;
170
+ beforeQuestionCount: number;
171
+ afterQuestionCount: number;
172
+ newQuestionCount: number;
173
+ expandedBefore: boolean;
174
+ expandedAfter: boolean;
175
+ answerObserved: boolean;
176
+ sourceCount: number;
177
+ errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
144
178
  }
145
179
  type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
146
180
  /** Material already observed before PAA expansion completes. It is checkpointed
@@ -164,6 +198,7 @@ interface PaaProgressSnapshot {
164
198
  records: FlatRow[];
165
199
  lifecycle?: PaaLifecycleDiagnostics;
166
200
  material?: PaaProgressMaterial;
201
+ interactions?: PaaInteractionObservation[];
167
202
  }
168
203
  interface HarvestDiagnostics {
169
204
  completionStatus: HarvestCompletionStatus;
package/dist/index.js CHANGED
@@ -1,12 +1,12 @@
1
1
  import {
2
2
  harvest
3
- } from "./chunk-WEEAXPWK.js";
3
+ } from "./chunk-KNGTQD6U.js";
4
4
  import "./chunk-OM7HVEJ3.js";
5
5
  import {
6
6
  recordVendorUsage
7
- } from "./chunk-NGLZ6UJR.js";
7
+ } from "./chunk-5UAFBKJR.js";
8
8
  import "./chunk-OWF2JJKN.js";
9
- import "./chunk-OU4FV2RP.js";
9
+ import "./chunk-YXNDOQXN.js";
10
10
 
11
11
  // src/video/VideoGenerator.ts
12
12
  import { execSync as execSync2 } from "child_process";
@@ -30,8 +30,8 @@ import {
30
30
  saveLeadListEnrichmentRowResult,
31
31
  setInitializingLeadListEnrichmentConcurrencyLock,
32
32
  stableCanonicalJson
33
- } from "./chunk-KAGHJQFO.js";
34
- import "./chunk-OU4FV2RP.js";
33
+ } from "./chunk-EQGTEHLZ.js";
34
+ import "./chunk-YXNDOQXN.js";
35
35
  export {
36
36
  LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
37
37
  LEAD_LIST_RUNNING_STALE_MINUTES,
@@ -13,9 +13,9 @@ import {
13
13
  parseHostedCensusPlacesCsv,
14
14
  parseHostedZipGroupsCsv,
15
15
  queryHostedLocationMarkets
16
- } from "./chunk-OUBH6XMY.js";
16
+ } from "./chunk-K3Z5AQYE.js";
17
17
  import "./chunk-RJ7JVYKU.js";
18
- import "./chunk-OU4FV2RP.js";
18
+ import "./chunk-YXNDOQXN.js";
19
19
  export {
20
20
  HOSTED_CENSUS_DATASET_PREFIX,
21
21
  HOSTED_ZIP_DATASET_KIND,