mcp-scraper 0.79.2 → 0.79.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +1 -1
- package/dist/{analytics-repository-VEUDEPWI.js → analytics-repository-YOY5KMOD.js} +3 -3
- package/dist/bin/api-server.js +4 -4
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +7 -7
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +7 -7
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-LB2GNACG.js → chunk-3423R23T.js} +3 -3
- package/dist/{chunk-NGLZ6UJR.js → chunk-5UAFBKJR.js} +1 -1
- package/dist/{chunk-B6L73AO4.js → chunk-6HAV7LCE.js} +3 -1
- package/dist/{chunk-JECPQBLZ.js → chunk-7UZAVM4J.js} +138 -11
- package/dist/{chunk-Q5L2OCOQ.js → chunk-C5Z4OFKW.js} +1 -1
- package/dist/{chunk-KAGHJQFO.js → chunk-EQGTEHLZ.js} +1 -1
- package/dist/{chunk-BTQ23KCT.js → chunk-F5GQJWZU.js} +1 -1
- package/dist/{chunk-DUWZQSSJ.js → chunk-HEUMDX4I.js} +1 -1
- package/dist/{chunk-2RTELL6M.js → chunk-IHYG52ZW.js} +1 -1
- package/dist/{chunk-OUBH6XMY.js → chunk-K3Z5AQYE.js} +1 -1
- package/dist/{chunk-WEEAXPWK.js → chunk-KNGTQD6U.js} +207 -36
- package/dist/{chunk-6FL7ZZCF.js → chunk-MVB25X55.js} +1 -1
- package/dist/{chunk-47ZCUBM5.js → chunk-NVUKO5NN.js} +12 -1
- package/dist/{chunk-XUM2LEGU.js → chunk-TCYZ7HMU.js} +3 -3
- package/dist/{chunk-SL7YVA4S.js → chunk-YBR32LOD.js} +1 -1
- package/dist/{chunk-OU4FV2RP.js → chunk-YXNDOQXN.js} +49 -0
- package/dist/{db-TA567JLA.js → db-Z34LPZNR.js} +5 -1
- package/dist/{extract-bundle-I2GNC25L.js → extract-bundle-S3ZGNFML.js} +3 -3
- package/dist/{gmail-service-GGQVJUC3.js → gmail-service-YFS2OV4S.js} +4 -4
- package/dist/index.cjs +205 -34
- package/dist/index.d.cts +35 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +3 -3
- package/dist/{lead-list-enrichment-repository-5LEWW4X7.js → lead-list-enrichment-repository-36RPVV6N.js} +2 -2
- package/dist/{location-data-repository-AHKDXTEQ.js → location-data-repository-WPRG62GE.js} +2 -2
- package/dist/{server-5OSHEEVT.js → server-MDWFOUTS.js} +73 -28
- package/dist/{site-extract-repository-KF736IR6.js → site-extract-repository-WAJMIDPJ.js} +2 -2
- package/dist/{worker-2TYCYFPM.js → worker-3LUFQWWJ.js} +5 -5
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -2807,17 +2807,22 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
|
|
|
2807
2807
|
// src/extractor/PAAExtractor.ts
|
|
2808
2808
|
var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
|
|
2809
2809
|
var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
|
|
2810
|
+
var PAA_CLICK_MID_CONFIRMATION_MS = 1e3;
|
|
2810
2811
|
var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
|
|
2812
|
+
var PAA_CLICK_DISPATCH_TIMEOUT_MS = 900;
|
|
2811
2813
|
var PAA_FRONTIER_BATCH_SIZE = 3;
|
|
2814
|
+
var MAX_RETAINED_INTERACTION_OBSERVATIONS = 250;
|
|
2812
2815
|
var PAAExtractor = class {
|
|
2813
|
-
constructor(driver, reporter, onPaaProgress) {
|
|
2816
|
+
constructor(driver, reporter, onPaaProgress, onProviderSession) {
|
|
2814
2817
|
this.driver = driver;
|
|
2815
2818
|
this.reporter = reporter;
|
|
2816
2819
|
this.onPaaProgress = onPaaProgress;
|
|
2820
|
+
this.onProviderSession = onProviderSession;
|
|
2817
2821
|
}
|
|
2818
2822
|
driver;
|
|
2819
2823
|
reporter;
|
|
2820
2824
|
onPaaProgress;
|
|
2825
|
+
onProviderSession;
|
|
2821
2826
|
completeness = {
|
|
2822
2827
|
paaWithoutAnswer: 0,
|
|
2823
2828
|
paaWithoutSource: 0,
|
|
@@ -2832,6 +2837,7 @@ var PAAExtractor = class {
|
|
|
2832
2837
|
clickedQuestions = /* @__PURE__ */ new Set();
|
|
2833
2838
|
paaLifecycle = this.emptyPaaLifecycle(0);
|
|
2834
2839
|
checkpointMaterial = null;
|
|
2840
|
+
interactionObservations = [];
|
|
2835
2841
|
emptyPaaLifecycle(requestedQuestions) {
|
|
2836
2842
|
return {
|
|
2837
2843
|
requestedQuestions,
|
|
@@ -2844,7 +2850,12 @@ var PAAExtractor = class {
|
|
|
2844
2850
|
knownQuestions: 0,
|
|
2845
2851
|
processedQuestions: 0,
|
|
2846
2852
|
failedInteractions: 0,
|
|
2847
|
-
noGrowthConfirmations: 0
|
|
2853
|
+
noGrowthConfirmations: 0,
|
|
2854
|
+
visibleControls: 0,
|
|
2855
|
+
eligibleControls: 0,
|
|
2856
|
+
dispatchedInteractions: 0,
|
|
2857
|
+
confirmedInteractions: 0,
|
|
2858
|
+
unconfirmedInteractions: 0
|
|
2848
2859
|
};
|
|
2849
2860
|
}
|
|
2850
2861
|
emptyLinkDiagnostics() {
|
|
@@ -3106,7 +3117,11 @@ var PAAExtractor = class {
|
|
|
3106
3117
|
phase,
|
|
3107
3118
|
records: this.collectedRows.map((row) => ({ ...row })),
|
|
3108
3119
|
lifecycle: { ...this.paaLifecycle },
|
|
3109
|
-
...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
|
|
3120
|
+
...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {},
|
|
3121
|
+
interactions: this.interactionObservations.map((observation) => ({
|
|
3122
|
+
...observation,
|
|
3123
|
+
confirmationSignals: [...observation.confirmationSignals]
|
|
3124
|
+
}))
|
|
3110
3125
|
};
|
|
3111
3126
|
try {
|
|
3112
3127
|
await this.onPaaProgress(snapshot);
|
|
@@ -3187,6 +3202,7 @@ var PAAExtractor = class {
|
|
|
3187
3202
|
const seenQs = /* @__PURE__ */ new Set();
|
|
3188
3203
|
const orderedQs = [];
|
|
3189
3204
|
this.clickedQuestions = /* @__PURE__ */ new Set();
|
|
3205
|
+
this.interactionObservations = [];
|
|
3190
3206
|
const abandonedNeverReclick = /* @__PURE__ */ new Set();
|
|
3191
3207
|
const clickAttempts = /* @__PURE__ */ new Map();
|
|
3192
3208
|
const processedQuestions = /* @__PURE__ */ new Set();
|
|
@@ -3194,6 +3210,7 @@ var PAAExtractor = class {
|
|
|
3194
3210
|
const capturedItems = /* @__PURE__ */ new Map();
|
|
3195
3211
|
const MAX_CLICK_ATTEMPTS = 2;
|
|
3196
3212
|
const EXHAUSTION_CONFIRMATIONS = 3;
|
|
3213
|
+
let batchNumber = 0;
|
|
3197
3214
|
const itemArgs = {
|
|
3198
3215
|
sel: PAASelectors.item,
|
|
3199
3216
|
dataQ: PAASelectors.itemDataQ,
|
|
@@ -3203,12 +3220,30 @@ var PAAExtractor = class {
|
|
|
3203
3220
|
expandedClass: PAASelectors.expandedClass,
|
|
3204
3221
|
clickTarget: PAASelectors.clickTarget,
|
|
3205
3222
|
answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
|
|
3206
|
-
questionsOnly: options.questionsOnly
|
|
3223
|
+
questionsOnly: options.questionsOnly,
|
|
3224
|
+
sourceSel: "a[href]"
|
|
3207
3225
|
};
|
|
3208
3226
|
const readItemStates = () => page.evaluate(
|
|
3209
3227
|
({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
|
|
3228
|
+
const cleanVisibleMaterial = (element) => {
|
|
3229
|
+
if (!element) return "";
|
|
3230
|
+
const parts = [];
|
|
3231
|
+
for (const node of element.childNodes) {
|
|
3232
|
+
if (node.nodeType === Node.TEXT_NODE) {
|
|
3233
|
+
const text2 = node.textContent?.trim();
|
|
3234
|
+
if (text2) parts.push(text2);
|
|
3235
|
+
continue;
|
|
3236
|
+
}
|
|
3237
|
+
if (node.nodeType !== Node.ELEMENT_NODE) continue;
|
|
3238
|
+
const child = node;
|
|
3239
|
+
if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
|
|
3240
|
+
const text = cleanVisibleMaterial(child);
|
|
3241
|
+
if (text) parts.push(text);
|
|
3242
|
+
}
|
|
3243
|
+
return parts.join(" ").replace(/\s+/g, " ").trim();
|
|
3244
|
+
};
|
|
3210
3245
|
return Array.from(document.querySelectorAll(sel)).map((el, index) => {
|
|
3211
|
-
const answerText = (el.querySelector(answerSel)
|
|
3246
|
+
const answerText = cleanVisibleMaterial(el.querySelector(answerSel)).replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
3212
3247
|
const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
|
|
3213
3248
|
const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
|
|
3214
3249
|
return {
|
|
@@ -3221,34 +3256,133 @@ var PAAExtractor = class {
|
|
|
3221
3256
|
},
|
|
3222
3257
|
itemArgs
|
|
3223
3258
|
);
|
|
3224
|
-
const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
|
|
3259
|
+
const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded, controlAttempt, batchNumber2) => {
|
|
3225
3260
|
const clickStartedMs = Date.now();
|
|
3261
|
+
const baseObservation = {
|
|
3262
|
+
id: `${batchNumber2}:${questionIdFor(q)}:${controlAttempt}`,
|
|
3263
|
+
questionId: questionIdFor(q),
|
|
3264
|
+
question: q,
|
|
3265
|
+
batchNumber: batchNumber2,
|
|
3266
|
+
controlAttempt,
|
|
3267
|
+
observedAt: new Date(clickStartedMs).toISOString(),
|
|
3268
|
+
beforeQuestionCount: beforeCount,
|
|
3269
|
+
expandedBefore: wasExpanded
|
|
3270
|
+
};
|
|
3226
3271
|
const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
|
|
3272
|
+
let dispatchErrorCode = null;
|
|
3227
3273
|
const clicked = clickTarget ? await (async () => {
|
|
3228
3274
|
if (wasExpanded) {
|
|
3229
|
-
await clickTarget.click({ timeout:
|
|
3230
|
-
await page.waitForTimeout(
|
|
3275
|
+
await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
|
|
3276
|
+
await page.waitForTimeout(120);
|
|
3231
3277
|
}
|
|
3232
|
-
await clickTarget.click({ timeout:
|
|
3278
|
+
await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
|
|
3233
3279
|
return true;
|
|
3234
|
-
})().catch(() =>
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3280
|
+
})().catch((error) => {
|
|
3281
|
+
dispatchErrorCode = error instanceof Error && (error.name === "TimeoutError" || /timeout/i.test(error.message)) ? "click_ack_timeout" : "click_failed";
|
|
3282
|
+
return false;
|
|
3283
|
+
}) : false;
|
|
3284
|
+
const dispatchDurationMs = Date.now() - clickStartedMs;
|
|
3285
|
+
if (!clicked && dispatchErrorCode !== "click_ack_timeout") {
|
|
3286
|
+
const observation2 = {
|
|
3287
|
+
...baseObservation,
|
|
3288
|
+
dispatched: false,
|
|
3289
|
+
outcome: "dispatch_failed",
|
|
3290
|
+
confirmationSignals: [],
|
|
3291
|
+
confirmationCheckpointMs: null,
|
|
3292
|
+
confirmedAtMs: null,
|
|
3293
|
+
dispatchDurationMs,
|
|
3294
|
+
totalDurationMs: dispatchDurationMs,
|
|
3295
|
+
afterQuestionCount: beforeCount,
|
|
3296
|
+
newQuestionCount: 0,
|
|
3297
|
+
expandedAfter: wasExpanded,
|
|
3298
|
+
answerObserved: false,
|
|
3299
|
+
sourceCount: 0,
|
|
3300
|
+
errorCode: itemIndex < 0 ? "control_missing" : dispatchErrorCode ?? "click_failed"
|
|
3301
|
+
};
|
|
3302
|
+
return { dispatched: false, confirmed: false, observation: observation2 };
|
|
3303
|
+
}
|
|
3304
|
+
let latest = {
|
|
3305
|
+
exists: true,
|
|
3306
|
+
expanded: wasExpanded,
|
|
3307
|
+
answerReady: false,
|
|
3308
|
+
sourceCount: 0,
|
|
3309
|
+
questionCount: beforeCount
|
|
3310
|
+
};
|
|
3311
|
+
let signals = [];
|
|
3312
|
+
let confirmationCheckpointMs = null;
|
|
3313
|
+
let confirmedAtMs = null;
|
|
3314
|
+
let confirmed = false;
|
|
3315
|
+
for (const checkpointMs of [PAA_CLICK_EARLIEST_CONFIRMATION_MS, PAA_CLICK_MID_CONFIRMATION_MS, PAA_CLICK_ATTEMPT_BUDGET_MS]) {
|
|
3316
|
+
const remainingMs = clickStartedMs + checkpointMs - Date.now();
|
|
3317
|
+
if (remainingMs > 0) await page.waitForTimeout(remainingMs);
|
|
3318
|
+
latest = await page.evaluate(
|
|
3319
|
+
({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, sourceSel, target }) => {
|
|
3320
|
+
const cleanVisibleMaterial = (element) => {
|
|
3321
|
+
if (!element) return "";
|
|
3322
|
+
const parts = [];
|
|
3323
|
+
for (const node of element.childNodes) {
|
|
3324
|
+
if (node.nodeType === Node.TEXT_NODE) {
|
|
3325
|
+
const text2 = node.textContent?.trim();
|
|
3326
|
+
if (text2) parts.push(text2);
|
|
3327
|
+
continue;
|
|
3328
|
+
}
|
|
3329
|
+
if (node.nodeType !== Node.ELEMENT_NODE) continue;
|
|
3330
|
+
const child = node;
|
|
3331
|
+
if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
|
|
3332
|
+
const text = cleanVisibleMaterial(child);
|
|
3333
|
+
if (text) parts.push(text);
|
|
3334
|
+
}
|
|
3335
|
+
return parts.join(" ").replace(/\s+/g, " ").trim();
|
|
3336
|
+
};
|
|
3337
|
+
const items = Array.from(document.querySelectorAll(sel));
|
|
3338
|
+
const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
|
|
3339
|
+
if (!item) return { exists: false, expanded: false, answerReady: false, sourceCount: 0, questionCount: items.length };
|
|
3340
|
+
const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
|
|
3341
|
+
const answerElement = item.querySelector(answerSel);
|
|
3342
|
+
const answerText = cleanVisibleMaterial(answerElement);
|
|
3343
|
+
const confirmationText = answerText.replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
|
|
3344
|
+
const answerReady = confirmationText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(confirmationText);
|
|
3345
|
+
return {
|
|
3346
|
+
exists: true,
|
|
3347
|
+
expanded,
|
|
3348
|
+
answerReady,
|
|
3349
|
+
sourceCount: answerReady ? answerElement?.querySelectorAll(sourceSel).length ?? 0 : 0,
|
|
3350
|
+
questionCount: items.length
|
|
3351
|
+
};
|
|
3352
|
+
},
|
|
3353
|
+
{ ...itemArgs, target: q }
|
|
3354
|
+
).catch(() => latest);
|
|
3355
|
+
signals = [
|
|
3356
|
+
...latest.expanded ? ["expanded"] : [],
|
|
3357
|
+
...latest.answerReady ? ["answer"] : [],
|
|
3358
|
+
...latest.sourceCount > 0 ? ["source"] : [],
|
|
3359
|
+
...latest.questionCount > beforeCount ? ["question_growth"] : []
|
|
3360
|
+
];
|
|
3361
|
+
if (latest.answerReady || latest.sourceCount > 0 || latest.questionCount > beforeCount) {
|
|
3362
|
+
confirmed = true;
|
|
3363
|
+
confirmationCheckpointMs = checkpointMs;
|
|
3364
|
+
confirmedAtMs = Date.now() - clickStartedMs;
|
|
3365
|
+
break;
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
const totalDurationMs = Date.now() - clickStartedMs;
|
|
3369
|
+
const observation = {
|
|
3370
|
+
...baseObservation,
|
|
3371
|
+
dispatched: clicked,
|
|
3372
|
+
outcome: confirmed ? "confirmed" : clicked ? "unconfirmed" : "dispatch_failed",
|
|
3373
|
+
confirmationSignals: signals,
|
|
3374
|
+
confirmationCheckpointMs,
|
|
3375
|
+
confirmedAtMs,
|
|
3376
|
+
dispatchDurationMs,
|
|
3377
|
+
totalDurationMs,
|
|
3378
|
+
afterQuestionCount: latest.questionCount,
|
|
3379
|
+
newQuestionCount: Math.max(0, latest.questionCount - beforeCount),
|
|
3380
|
+
expandedAfter: latest.expanded,
|
|
3381
|
+
answerObserved: latest.answerReady,
|
|
3382
|
+
sourceCount: latest.sourceCount,
|
|
3383
|
+
errorCode: !clicked ? dispatchErrorCode : confirmed ? null : latest.exists ? "confirmation_timeout" : "control_missing"
|
|
3384
|
+
};
|
|
3385
|
+
return { dispatched: clicked, confirmed, observation };
|
|
3252
3386
|
};
|
|
3253
3387
|
let round = 0;
|
|
3254
3388
|
expansion: while (true) {
|
|
@@ -3279,7 +3413,8 @@ var PAAExtractor = class {
|
|
|
3279
3413
|
targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
|
|
3280
3414
|
knownQuestions: orderedQs.length,
|
|
3281
3415
|
processedQuestions: processedQuestions.size,
|
|
3282
|
-
failedInteractions: failedInteractions.size
|
|
3416
|
+
failedInteractions: failedInteractions.size,
|
|
3417
|
+
visibleControls: states.length
|
|
3283
3418
|
};
|
|
3284
3419
|
if (discovered) await this.emitProgress("discovered");
|
|
3285
3420
|
if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
|
|
@@ -3327,6 +3462,7 @@ var PAAExtractor = class {
|
|
|
3327
3462
|
const kept = new Set(orderedQs.slice(0, options.maxQuestions));
|
|
3328
3463
|
const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
|
|
3329
3464
|
const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
|
|
3465
|
+
this.paaLifecycle = { ...this.paaLifecycle, eligibleControls: clickable.length };
|
|
3330
3466
|
if (targets.length === 0) {
|
|
3331
3467
|
if (seenQs.size >= options.maxQuestions) {
|
|
3332
3468
|
this.paaLifecycle = {
|
|
@@ -3376,6 +3512,7 @@ var PAAExtractor = class {
|
|
|
3376
3512
|
break;
|
|
3377
3513
|
}
|
|
3378
3514
|
await this.throwIfCaptcha(page, "Google PAA expansion");
|
|
3515
|
+
batchNumber += 1;
|
|
3379
3516
|
for (const target of targets) {
|
|
3380
3517
|
this.reporter.onDepth(++round);
|
|
3381
3518
|
if ((clickAttempts.get(target.q) ?? 0) > 0) {
|
|
@@ -3388,10 +3525,21 @@ var PAAExtractor = class {
|
|
|
3388
3525
|
}
|
|
3389
3526
|
const expansionResults = await Promise.all(targets.map(async (target) => ({
|
|
3390
3527
|
target,
|
|
3391
|
-
result: await expandOneItemSerially(
|
|
3528
|
+
result: await expandOneItemSerially(
|
|
3529
|
+
target.q,
|
|
3530
|
+
states.length,
|
|
3531
|
+
target.index,
|
|
3532
|
+
target.expanded,
|
|
3533
|
+
clickAttempts.get(target.q) ?? 1,
|
|
3534
|
+
batchNumber
|
|
3535
|
+
)
|
|
3392
3536
|
})));
|
|
3393
3537
|
for (const { target, result } of expansionResults) {
|
|
3394
|
-
if (result.dispatched) this.clickedQuestions.add(target.q);
|
|
3538
|
+
if (result.dispatched || result.confirmed) this.clickedQuestions.add(target.q);
|
|
3539
|
+
this.interactionObservations.push(result.observation);
|
|
3540
|
+
}
|
|
3541
|
+
if (this.interactionObservations.length > MAX_RETAINED_INTERACTION_OBSERVATIONS) {
|
|
3542
|
+
this.interactionObservations.splice(0, this.interactionObservations.length - MAX_RETAINED_INTERACTION_OBSERVATIONS);
|
|
3395
3543
|
}
|
|
3396
3544
|
if (!options.questionsOnly) {
|
|
3397
3545
|
const targetQuestions = new Set(targets.map((target) => target.q));
|
|
@@ -3417,7 +3565,10 @@ var PAAExtractor = class {
|
|
|
3417
3565
|
...this.paaLifecycle,
|
|
3418
3566
|
processedQuestions: processedQuestions.size,
|
|
3419
3567
|
failedInteractions: failedInteractions.size,
|
|
3420
|
-
noGrowthConfirmations: 0
|
|
3568
|
+
noGrowthConfirmations: 0,
|
|
3569
|
+
dispatchedInteractions: this.interactionObservations.filter((value) => value.dispatched).length,
|
|
3570
|
+
confirmedInteractions: this.interactionObservations.filter((value) => value.outcome === "confirmed").length,
|
|
3571
|
+
unconfirmedInteractions: this.interactionObservations.filter((value) => value.outcome !== "confirmed").length
|
|
3421
3572
|
};
|
|
3422
3573
|
await this.emitProgress("click_observed");
|
|
3423
3574
|
}
|
|
@@ -4011,6 +4162,18 @@ var PAAExtractor = class {
|
|
|
4011
4162
|
try {
|
|
4012
4163
|
this.throwIfAborted(signal);
|
|
4013
4164
|
await this.driver.launch(config);
|
|
4165
|
+
const providerSession = this.driver.getProviderSessionMetadata?.();
|
|
4166
|
+
if (providerSession?.providerSessionId && this.onProviderSession) {
|
|
4167
|
+
try {
|
|
4168
|
+
await this.onProviderSession(providerSession);
|
|
4169
|
+
} catch (err) {
|
|
4170
|
+
console.warn(JSON.stringify({
|
|
4171
|
+
event: "paa_provider_session_sink_failed",
|
|
4172
|
+
provider: providerSession.provider,
|
|
4173
|
+
message: err instanceof Error ? err.message : String(err)
|
|
4174
|
+
}));
|
|
4175
|
+
}
|
|
4176
|
+
}
|
|
4014
4177
|
this.throwIfAborted(signal);
|
|
4015
4178
|
const locationResolution = options.location ? resolveLocation(options.location) : null;
|
|
4016
4179
|
const canonicalLocation = options.location ? locationResolution?.canonical ?? options.location : null;
|
|
@@ -5142,10 +5305,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
|
|
|
5142
5305
|
}));
|
|
5143
5306
|
}
|
|
5144
5307
|
}
|
|
5145
|
-
async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
|
|
5308
|
+
async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress, onProviderSession) {
|
|
5146
5309
|
const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
|
|
5147
5310
|
const reporter = new ProgressReporter();
|
|
5148
|
-
const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
|
|
5311
|
+
const extractor = new PAAExtractor(driver, reporter, onPaaProgress, onProviderSession);
|
|
5149
5312
|
if (signal?.aborted) {
|
|
5150
5313
|
return {
|
|
5151
5314
|
result: null,
|
|
@@ -5290,12 +5453,20 @@ async function harvest(rawOptions) {
|
|
|
5290
5453
|
max_questions: attemptOptions.maxQuestions
|
|
5291
5454
|
}));
|
|
5292
5455
|
const baseCtx = currentCostContext();
|
|
5456
|
+
const onProviderSession = (metadata) => emitAttemptEvent(onAttemptEvent, {
|
|
5457
|
+
type: "provider_session_started",
|
|
5458
|
+
attemptNumber,
|
|
5459
|
+
maxAttempts,
|
|
5460
|
+
provider: metadata.provider,
|
|
5461
|
+
providerSessionId: metadata.providerSessionId,
|
|
5462
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5463
|
+
});
|
|
5293
5464
|
const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
|
|
5294
5465
|
const attempt = baseCtx ? await runWithCostContext({
|
|
5295
5466
|
...baseCtx,
|
|
5296
5467
|
forceHeadless: allowHeadlessTest,
|
|
5297
5468
|
forceHeadful: !allowHeadlessTest
|
|
5298
|
-
}, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
|
|
5469
|
+
}, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession);
|
|
5299
5470
|
if (attempt.error) {
|
|
5300
5471
|
const err = attempt.error;
|
|
5301
5472
|
const outcome2 = classifyAttemptError(err);
|
package/dist/index.d.cts
CHANGED
|
@@ -141,6 +141,40 @@ interface PaaLifecycleDiagnostics {
|
|
|
141
141
|
processedQuestions: number;
|
|
142
142
|
failedInteractions: number;
|
|
143
143
|
noGrowthConfirmations: number;
|
|
144
|
+
visibleControls?: number;
|
|
145
|
+
eligibleControls?: number;
|
|
146
|
+
dispatchedInteractions?: number;
|
|
147
|
+
confirmedInteractions?: number;
|
|
148
|
+
unconfirmedInteractions?: number;
|
|
149
|
+
}
|
|
150
|
+
type PaaInteractionOutcome = 'confirmed' | 'unconfirmed' | 'dispatch_failed';
|
|
151
|
+
type PaaInteractionSignal = 'expanded' | 'answer' | 'source' | 'question_growth';
|
|
152
|
+
/** One bounded, auditable interaction with a rendered PAA control. Question
|
|
153
|
+
* growth is recorded separately from control-local answer/expanded signals so
|
|
154
|
+
* a multi-control batch never pretends it can attribute shared DOM growth to a
|
|
155
|
+
* specific click with more certainty than the browser exposed. */
|
|
156
|
+
interface PaaInteractionObservation {
|
|
157
|
+
id: string;
|
|
158
|
+
questionId: string;
|
|
159
|
+
question: string;
|
|
160
|
+
batchNumber: number;
|
|
161
|
+
controlAttempt: number;
|
|
162
|
+
observedAt: string;
|
|
163
|
+
dispatched: boolean;
|
|
164
|
+
outcome: PaaInteractionOutcome;
|
|
165
|
+
confirmationSignals: PaaInteractionSignal[];
|
|
166
|
+
confirmationCheckpointMs: 700 | 1000 | 1400 | null;
|
|
167
|
+
confirmedAtMs: number | null;
|
|
168
|
+
dispatchDurationMs: number;
|
|
169
|
+
totalDurationMs: number;
|
|
170
|
+
beforeQuestionCount: number;
|
|
171
|
+
afterQuestionCount: number;
|
|
172
|
+
newQuestionCount: number;
|
|
173
|
+
expandedBefore: boolean;
|
|
174
|
+
expandedAfter: boolean;
|
|
175
|
+
answerObserved: boolean;
|
|
176
|
+
sourceCount: number;
|
|
177
|
+
errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
|
|
144
178
|
}
|
|
145
179
|
type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
|
|
146
180
|
/** Material already observed before PAA expansion completes. It is checkpointed
|
|
@@ -164,6 +198,7 @@ interface PaaProgressSnapshot {
|
|
|
164
198
|
records: FlatRow[];
|
|
165
199
|
lifecycle?: PaaLifecycleDiagnostics;
|
|
166
200
|
material?: PaaProgressMaterial;
|
|
201
|
+
interactions?: PaaInteractionObservation[];
|
|
167
202
|
}
|
|
168
203
|
interface HarvestDiagnostics {
|
|
169
204
|
completionStatus: HarvestCompletionStatus;
|
package/dist/index.d.ts
CHANGED
|
@@ -141,6 +141,40 @@ interface PaaLifecycleDiagnostics {
|
|
|
141
141
|
processedQuestions: number;
|
|
142
142
|
failedInteractions: number;
|
|
143
143
|
noGrowthConfirmations: number;
|
|
144
|
+
visibleControls?: number;
|
|
145
|
+
eligibleControls?: number;
|
|
146
|
+
dispatchedInteractions?: number;
|
|
147
|
+
confirmedInteractions?: number;
|
|
148
|
+
unconfirmedInteractions?: number;
|
|
149
|
+
}
|
|
150
|
+
type PaaInteractionOutcome = 'confirmed' | 'unconfirmed' | 'dispatch_failed';
|
|
151
|
+
type PaaInteractionSignal = 'expanded' | 'answer' | 'source' | 'question_growth';
|
|
152
|
+
/** One bounded, auditable interaction with a rendered PAA control. Question
|
|
153
|
+
* growth is recorded separately from control-local answer/expanded signals so
|
|
154
|
+
* a multi-control batch never pretends it can attribute shared DOM growth to a
|
|
155
|
+
* specific click with more certainty than the browser exposed. */
|
|
156
|
+
interface PaaInteractionObservation {
|
|
157
|
+
id: string;
|
|
158
|
+
questionId: string;
|
|
159
|
+
question: string;
|
|
160
|
+
batchNumber: number;
|
|
161
|
+
controlAttempt: number;
|
|
162
|
+
observedAt: string;
|
|
163
|
+
dispatched: boolean;
|
|
164
|
+
outcome: PaaInteractionOutcome;
|
|
165
|
+
confirmationSignals: PaaInteractionSignal[];
|
|
166
|
+
confirmationCheckpointMs: 700 | 1000 | 1400 | null;
|
|
167
|
+
confirmedAtMs: number | null;
|
|
168
|
+
dispatchDurationMs: number;
|
|
169
|
+
totalDurationMs: number;
|
|
170
|
+
beforeQuestionCount: number;
|
|
171
|
+
afterQuestionCount: number;
|
|
172
|
+
newQuestionCount: number;
|
|
173
|
+
expandedBefore: boolean;
|
|
174
|
+
expandedAfter: boolean;
|
|
175
|
+
answerObserved: boolean;
|
|
176
|
+
sourceCount: number;
|
|
177
|
+
errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
|
|
144
178
|
}
|
|
145
179
|
type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
|
|
146
180
|
/** Material already observed before PAA expansion completes. It is checkpointed
|
|
@@ -164,6 +198,7 @@ interface PaaProgressSnapshot {
|
|
|
164
198
|
records: FlatRow[];
|
|
165
199
|
lifecycle?: PaaLifecycleDiagnostics;
|
|
166
200
|
material?: PaaProgressMaterial;
|
|
201
|
+
interactions?: PaaInteractionObservation[];
|
|
167
202
|
}
|
|
168
203
|
interface HarvestDiagnostics {
|
|
169
204
|
completionStatus: HarvestCompletionStatus;
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
harvest
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-KNGTQD6U.js";
|
|
4
4
|
import "./chunk-OM7HVEJ3.js";
|
|
5
5
|
import {
|
|
6
6
|
recordVendorUsage
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-5UAFBKJR.js";
|
|
8
8
|
import "./chunk-OWF2JJKN.js";
|
|
9
|
-
import "./chunk-
|
|
9
|
+
import "./chunk-YXNDOQXN.js";
|
|
10
10
|
|
|
11
11
|
// src/video/VideoGenerator.ts
|
|
12
12
|
import { execSync as execSync2 } from "child_process";
|
|
@@ -30,8 +30,8 @@ import {
|
|
|
30
30
|
saveLeadListEnrichmentRowResult,
|
|
31
31
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
32
32
|
stableCanonicalJson
|
|
33
|
-
} from "./chunk-
|
|
34
|
-
import "./chunk-
|
|
33
|
+
} from "./chunk-EQGTEHLZ.js";
|
|
34
|
+
import "./chunk-YXNDOQXN.js";
|
|
35
35
|
export {
|
|
36
36
|
LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
|
|
37
37
|
LEAD_LIST_RUNNING_STALE_MINUTES,
|
|
@@ -13,9 +13,9 @@ import {
|
|
|
13
13
|
parseHostedCensusPlacesCsv,
|
|
14
14
|
parseHostedZipGroupsCsv,
|
|
15
15
|
queryHostedLocationMarkets
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-K3Z5AQYE.js";
|
|
17
17
|
import "./chunk-RJ7JVYKU.js";
|
|
18
|
-
import "./chunk-
|
|
18
|
+
import "./chunk-YXNDOQXN.js";
|
|
19
19
|
export {
|
|
20
20
|
HOSTED_CENSUS_DATASET_PREFIX,
|
|
21
21
|
HOSTED_ZIP_DATASET_KIND,
|