@revclamp/google-ads-transparency-mcp 0.0.0-stage → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2208 @@
1
+ /**
2
+ * Apify Pay-per-Event Execution, Native Platform Adapter & Crash-Safe Billing State Machine (src/apify_actor.mjs)
3
+ *
4
+ * Enforces deterministic Pay-per-Event billing and delivery invariants:
5
+ * 1. Explicit Accounting Separation: Tracks `extractedCount`, `deliveredCount`,
6
+ * `billableCount` (simulated unit-economics count), and confirmed `chargedEventCount`
7
+ * (`0` in local/MCP unbilled mode; strictly validated integer in Apify billing mode).
8
+ * 2. Native Zero-Dependency Apify Platform Adapter (`createApifyPlatformHooks` & `loadApifyActorInput`):
9
+ * Wires Apify Key-Value Store (`INPUT` & `ADPULSE_BILLING_CHECKPOINT`), Dataset (`pushData`),
10
+ * and Pay-per-Event Charging (`POST /v2/actor-runs/:runId/charge` with `Idempotency-Key`)
11
+ * when running on Apify (`APIFY_IS_AT_HOME=1` / `APIFY_TOKEN` + `APIFY_ACTOR_RUN_ID`) or local storage.
12
+ * 3. Crash-Safe KVS Checkpoint State Machine (`inFlightIds` -> `chargedIds` -> `deliveredIds`
13
+ * or `unknownOutcomeIds`), deterministic prefix delivery when `eventChargeLimitReached` returns
14
+ * partial `chargedCount`, cursor rewind on mid-stream halt/limit, and honest zero-event telemetry.
15
+ */
16
+
17
+ import fs from 'node:fs';
18
+ import path from 'node:path';
19
+ import crypto from 'node:crypto';
20
+ import {
21
+ resolveAdvertiserEntities,
22
+ fetchAdvertiserCreatives,
23
+ getCreativeDetailsById,
24
+ createEmptyUntrustedAdContent,
25
+ computeRewoundCursorForDeliveredPrefix,
26
+ verifyEgressUrlSsrfSafe,
27
+ CREATIVE_ID_RE,
28
+ ADVERTISER_ID_RE,
29
+ DEFAULT_RUN_DEADLINE_MS,
30
+ } from './rpc_client.mjs';
31
+
32
+ export const ACTOR_PRICING_CONTRACT = Object.freeze({
33
+ eventName: 'ad-creative-scraped',
34
+ actorStartFeeUsdc: 0.001,
35
+ priceUsdc: 0.0012,
36
+ costCeilingUsdc: 0.0001,
37
+ platformFeeRate: 0.2,
38
+ netPriceUsdc: 0.00096,
39
+ minMarginMultiplier: 2.5,
40
+ computeRatePerGbHourUsdc: 0.3,
41
+ maxMemoryMb: 256,
42
+ maxTimeoutSec: 25,
43
+ chunkSize: 25,
44
+ checkpointKey: 'ADPULSE_BILLING_CHECKPOINT',
45
+ });
46
+
47
+ /**
48
+ * Computes content-addressed batch idempotency key (`sha256(runId + '\x1f' + sortedChunkIds)`).
49
+ *
50
+ * @param {string} runId
51
+ * @param {string[]} creativeIds
52
+ * @returns {string}
53
+ */
54
+ export function computeChunkIdempotencyKey(runId, creativeIds) {
55
+ const safeRunId = String(runId || 'default_run');
56
+ const sortedIds = [...creativeIds].map((id) => String(id)).sort();
57
+ return crypto
58
+ .createHash('sha256')
59
+ .update(`${safeRunId}\x1f${sortedIds.join('\x1f')}`, 'utf8')
60
+ .digest('hex');
61
+ }
62
+
63
+ /**
64
+ * Computes unit-economics and margin telemetry for a single Actor/MCP execution run.
65
+ * Uses Apify's standard `$0.30 / GB-hour` compute rate (`0.25 GB` at `256MB`).
66
+ * When `billableCount === 0`, returns `economicsStatus: 'NO_BILLABLE_EVENTS'` with
67
+ * `withinCostCeiling: false` and `marginPositive: false`.
68
+ *
69
+ * @param {object} params
70
+ * @param {number} params.billableCount
71
+ * @param {number} [params.chargedEventCount=0]
72
+ * @param {number} params.durationMs
73
+ * @param {number} [params.memoryMb=256]
74
+ * @param {number} [params.requestCount=1]
75
+ * @returns {object}
76
+ */
77
+ export function computeRunEconomicsTelemetry({
78
+ billableCount = 0,
79
+ chargedEventCount = 0,
80
+ durationMs = 100,
81
+ memoryMb =
82
+ Number(process.env.ACTOR_MEMORY_MBYTES || process.env.APIFY_MEMORY_MBYTES) ||
83
+ ACTOR_PRICING_CONTRACT.maxMemoryMb,
84
+ requestCount = 1,
85
+ unitPriceUsdc = ACTOR_PRICING_CONTRACT.priceUsdc,
86
+ actorStartFeeUsdc = ACTOR_PRICING_CONTRACT.actorStartFeeUsdc,
87
+ } = {}) {
88
+ const safeCount = Math.max(0, Math.floor(Number(billableCount) || 0));
89
+ const safeChargedCount = Math.max(0, Math.floor(Number(chargedEventCount) || 0));
90
+ const safeDurationMs = Math.max(1, Number(durationMs) || 1);
91
+ const safeMemoryMb = Math.min(1024, Math.max(64, Number(memoryMb) || 256));
92
+ const safeRequests = Math.max(1, Math.floor(Number(requestCount) || 1));
93
+ const resolvedUnitPriceUsdc =
94
+ Number.isFinite(Number(unitPriceUsdc)) && Number(unitPriceUsdc) > 0
95
+ ? Number(unitPriceUsdc)
96
+ : ACTOR_PRICING_CONTRACT.priceUsdc;
97
+ const resolvedNetPriceUsdc = Number(
98
+ (resolvedUnitPriceUsdc * (1 - ACTOR_PRICING_CONTRACT.platformFeeRate)).toFixed(6)
99
+ );
100
+ const resolvedStartFeeUsdc =
101
+ Number.isFinite(Number(actorStartFeeUsdc)) && Number(actorStartFeeUsdc) >= 0
102
+ ? Number(actorStartFeeUsdc)
103
+ : ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
104
+
105
+ // Apify compute rate: $0.30 per GB-hour; 256MB = 0.25 GB -> $0.075/hour -> $0.00002083/sec
106
+ const computeHours = safeDurationMs / 3600000;
107
+ const computeCostUsdc =
108
+ (safeMemoryMb / 1024) * computeHours * ACTOR_PRICING_CONTRACT.computeRatePerGbHourUsdc;
109
+ const networkOverheadUsdc = safeRequests * 0.0000002;
110
+ const totalRunCostUsdc = Number((computeCostUsdc + networkOverheadUsdc).toFixed(8));
111
+
112
+ const configuredFloorMultiplier = Number(
113
+ (resolvedNetPriceUsdc / ACTOR_PRICING_CONTRACT.costCeilingUsdc).toFixed(2)
114
+ );
115
+
116
+ if (safeCount === 0) {
117
+ return {
118
+ economicsStatus: 'NO_BILLABLE_EVENTS',
119
+ billableCount: 0,
120
+ chargedEventCount: safeChargedCount,
121
+ durationMs: safeDurationMs,
122
+ memoryMb: safeMemoryMb,
123
+ requestCount: safeRequests,
124
+ actorStartFeeUsdc: resolvedStartFeeUsdc,
125
+ totalRunCostUsdc,
126
+ actualCostPerEventUsdc: 0,
127
+ costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
128
+ withinCostCeiling: false,
129
+ grossRevenueUsdc: 0,
130
+ netRevenueUsdc: 0,
131
+ configuredFloorMultiplier,
132
+ effectiveMarginMultiplier: 0,
133
+ marginPositive: false,
134
+ };
135
+ }
136
+
137
+ const actualCostPerEventUsdc = Number((totalRunCostUsdc / safeCount).toFixed(8));
138
+ const grossRevenueUsdc = Number((safeCount * resolvedUnitPriceUsdc).toFixed(6));
139
+ const netRevenueUsdc = Number((safeCount * resolvedNetPriceUsdc).toFixed(6));
140
+ const effectiveMarginMultiplier =
141
+ actualCostPerEventUsdc > 0
142
+ ? Number((resolvedNetPriceUsdc / actualCostPerEventUsdc).toFixed(2))
143
+ : configuredFloorMultiplier;
144
+
145
+ const withinCostCeiling = actualCostPerEventUsdc <= ACTOR_PRICING_CONTRACT.costCeilingUsdc;
146
+ const marginPositive =
147
+ withinCostCeiling &&
148
+ configuredFloorMultiplier >= ACTOR_PRICING_CONTRACT.minMarginMultiplier;
149
+
150
+ return {
151
+ economicsStatus: marginPositive ? 'PROFITABLE' : 'COST_CEILING_EXCEEDED',
152
+ billableCount: safeCount,
153
+ chargedEventCount: safeChargedCount,
154
+ durationMs: safeDurationMs,
155
+ memoryMb: safeMemoryMb,
156
+ requestCount: safeRequests,
157
+ actorStartFeeUsdc: resolvedStartFeeUsdc,
158
+ totalRunCostUsdc,
159
+ actualCostPerEventUsdc,
160
+ costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
161
+ withinCostCeiling,
162
+ grossRevenueUsdc,
163
+ netRevenueUsdc,
164
+ configuredFloorMultiplier,
165
+ effectiveMarginMultiplier,
166
+ marginPositive,
167
+ };
168
+ }
169
+
170
+ /**
171
+ * Validates that an Apify API base URL is safe (`https://api.apify.com` by default).
172
+ * When `rawApiBaseUrl` is an internal HTTP endpoint on Apify Cloud and a valid HTTPS
173
+ * `fallbackPublicBaseUrl` (`APIFY_API_PUBLIC_BASE_URL`) is provided, uses the HTTPS public URL.
174
+ *
175
+ * @param {string} [rawApiBaseUrl]
176
+ * @param {string} [fallbackPublicBaseUrl]
177
+ * @returns {string}
178
+ */
179
+ function resolveSafeApifyApiBaseUrl(rawApiBaseUrl, fallbackPublicBaseUrl = undefined) {
180
+ const candidate = String(rawApiBaseUrl || fallbackPublicBaseUrl || 'https://api.apify.com')
181
+ .trim()
182
+ .replace(/\/+$/, '');
183
+ const ssrf = verifyEgressUrlSsrfSafe(candidate);
184
+ if (!ssrf.safe) {
185
+ if (fallbackPublicBaseUrl) {
186
+ const publicCandidate = String(fallbackPublicBaseUrl).trim().replace(/\/+$/, '');
187
+ const publicSsrf = verifyEgressUrlSsrfSafe(publicCandidate);
188
+ if (publicSsrf.safe) {
189
+ return publicCandidate;
190
+ }
191
+ }
192
+ throw new Error(`Invalid or unsafe APIFY_API_BASE_URL '${candidate}': ${ssrf.reason}`);
193
+ }
194
+ return candidate;
195
+ }
196
+
197
+ /**
198
+ * Loads Actor input JSON from CLI `--input-json`, `APIFY_INPUT_PATH`, local Apify storage
199
+ * (`$ACTOR_STORAGE_DIR` / `$APIFY_LOCAL_STORAGE_DIR/key_value_stores/default/INPUT.json`),
200
+ * or Apify Platform KVS REST API.
201
+ *
202
+ * @param {string[]} [cliArgs=[]]
203
+ * @param {object} [env=process.env]
204
+ * @param {Function} [fetchImpl=globalThis.fetch]
205
+ * @returns {Promise<object>}
206
+ */
207
+ export async function loadApifyActorInput(
208
+ cliArgs = [],
209
+ env = process.env,
210
+ fetchImpl = globalThis.fetch
211
+ ) {
212
+ const inputIdx = Array.isArray(cliArgs) ? cliArgs.indexOf('--input-json') : -1;
213
+ if (inputIdx !== -1 && cliArgs[inputIdx + 1]) {
214
+ const raw = fs.readFileSync(cliArgs[inputIdx + 1], 'utf8');
215
+ return JSON.parse(raw);
216
+ }
217
+
218
+ const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
219
+ const applyAtHomeSchemaDefaults = (parsedInput) => {
220
+ if (
221
+ isAtHome &&
222
+ parsedInput &&
223
+ typeof parsedInput === 'object' &&
224
+ !Array.isArray(parsedInput) &&
225
+ !Object.prototype.hasOwnProperty.call(parsedInput, 'query') &&
226
+ !Object.prototype.hasOwnProperty.call(parsedInput, 'advertiserId') &&
227
+ !Object.prototype.hasOwnProperty.call(parsedInput, 'pageToken')
228
+ ) {
229
+ return { query: 'Stripe', ...parsedInput };
230
+ }
231
+ return parsedInput;
232
+ };
233
+
234
+ if (env?.APIFY_INPUT_PATH && fs.existsSync(env.APIFY_INPUT_PATH)) {
235
+ const raw = fs.readFileSync(env.APIFY_INPUT_PATH, 'utf8');
236
+ return applyAtHomeSchemaDefaults(raw && raw.trim() ? JSON.parse(raw) : {});
237
+ }
238
+
239
+ const localStorageDir =
240
+ env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
241
+ if (localStorageDir) {
242
+ const kvsId =
243
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ||
244
+ env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID ||
245
+ 'default';
246
+ const inputKey = env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT';
247
+ for (const candidateName of [`${inputKey}.json`, inputKey]) {
248
+ const localInputPath = path.join(
249
+ localStorageDir,
250
+ 'key_value_stores',
251
+ kvsId,
252
+ candidateName
253
+ );
254
+ if (fs.existsSync(localInputPath)) {
255
+ const raw = fs.readFileSync(localInputPath, 'utf8');
256
+ return applyAtHomeSchemaDefaults(raw && raw.trim() ? JSON.parse(raw) : {});
257
+ }
258
+ }
259
+ }
260
+
261
+ const kvsId =
262
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
263
+ const token = env?.APIFY_TOKEN;
264
+ if (kvsId && token && typeof fetchImpl === 'function') {
265
+ const apiBase = resolveSafeApifyApiBaseUrl(
266
+ env?.APIFY_API_BASE_URL,
267
+ env?.APIFY_API_PUBLIC_BASE_URL
268
+ );
269
+ const inputKey = encodeURIComponent(
270
+ env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT'
271
+ );
272
+ const url = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${inputKey}`;
273
+ const controller = new AbortController();
274
+ const timer = setTimeout(() => controller.abort(), 4000);
275
+ try {
276
+ const res = await fetchImpl(url, {
277
+ method: 'GET',
278
+ headers: {
279
+ authorization: `Bearer ${token}`,
280
+ accept: 'application/json',
281
+ },
282
+ signal: controller.signal,
283
+ });
284
+ if (res.status === 200) {
285
+ const text = await res.text();
286
+ const parsed = text && text.trim() ? JSON.parse(text) : {};
287
+ return applyAtHomeSchemaDefaults(parsed);
288
+ }
289
+ if (res.status === 204 || res.status === 404) {
290
+ return applyAtHomeSchemaDefaults({});
291
+ }
292
+ return {};
293
+ } finally {
294
+ clearTimeout(timer);
295
+ }
296
+ }
297
+
298
+ return applyAtHomeSchemaDefaults({});
299
+ }
300
+
301
+ /**
302
+ * Creates native zero-dependency Apify Platform runtime hooks (`runId`, `isAtHome`,
303
+ * `loadCheckpoint`, `saveCheckpoint`, `listDeliveredIds`, `pushData`, `chargeEvent`, `saveOutput`)
304
+ * when Apify environment variables (`ACTOR_*` primary, `APIFY_*` fallback) or local
305
+ * storage directories are configured. Returns `{}` in pure local CLI/MCP mode.
306
+ *
307
+ * @param {object} [env=process.env]
308
+ * @param {Function} [fetchImpl=globalThis.fetch]
309
+ * @returns {object}
310
+ */
311
+ export function createApifyPlatformHooks(env = process.env, fetchImpl = globalThis.fetch) {
312
+ const rawRunId = env?.ACTOR_RUN_ID || env?.APIFY_ACTOR_RUN_ID;
313
+ const runId = rawRunId ? String(rawRunId).trim() : undefined;
314
+ const token = env?.APIFY_TOKEN ? String(env.APIFY_TOKEN).trim() : undefined;
315
+ const rawDatasetId = env?.ACTOR_DEFAULT_DATASET_ID || env?.APIFY_DEFAULT_DATASET_ID;
316
+ const datasetId = rawDatasetId ? String(rawDatasetId).trim() : undefined;
317
+ const rawKvsId =
318
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
319
+ const kvsId = rawKvsId ? String(rawKvsId).trim() : undefined;
320
+ const localStorageDir =
321
+ env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
322
+ const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
323
+
324
+ const hasCloudApi = Boolean((isAtHome || runId) && token && typeof fetchImpl === 'function');
325
+ if (!hasCloudApi && !localStorageDir) {
326
+ if (isAtHome) {
327
+ return { isAtHome: true, ...(runId ? { runId } : {}) };
328
+ }
329
+ return runId ? { runId } : {};
330
+ }
331
+
332
+ const hooks = {};
333
+ if (runId) {
334
+ hooks.runId = runId;
335
+ }
336
+ if (isAtHome || hasCloudApi) {
337
+ hooks.isAtHome = true;
338
+ }
339
+
340
+ if (hasCloudApi) {
341
+ const apiBase = resolveSafeApifyApiBaseUrl(
342
+ env?.APIFY_API_BASE_URL,
343
+ env?.APIFY_API_PUBLIC_BASE_URL
344
+ );
345
+ let pricingInfoChargeEvents = null;
346
+ const rawPricingInfoEnv = env?.APIFY_ACTOR_PRICING_INFO || env?.ACTOR_PRICING_INFO;
347
+ if (rawPricingInfoEnv) {
348
+ try {
349
+ const parsedPricing = JSON.parse(String(rawPricingInfoEnv));
350
+ const eventsObj = parsedPricing?.pricingPerEvent?.actorChargeEvents;
351
+ if (eventsObj && typeof eventsObj === 'object') {
352
+ pricingInfoChargeEvents = eventsObj;
353
+ }
354
+ } catch {
355
+ pricingInfoChargeEvents = null;
356
+ }
357
+ }
358
+ const rawMaxChargeStr =
359
+ env?.ACTOR_MAX_TOTAL_CHARGE_USD !== undefined && env?.ACTOR_MAX_TOTAL_CHARGE_USD !== null
360
+ ? String(env.ACTOR_MAX_TOTAL_CHARGE_USD).trim()
361
+ : '';
362
+ const maxTotalChargeUsdRaw = rawMaxChargeStr !== '' ? Number(rawMaxChargeStr) : NaN;
363
+ const hasMaxChargeUsd = Number.isFinite(maxTotalChargeUsdRaw) && maxTotalChargeUsdRaw >= 0;
364
+ const includeStartFeeFlag = String(env?.ACTOR_INCLUDE_START_FEE ?? '1')
365
+ .trim()
366
+ .toLowerCase();
367
+ const includeStartFee = includeStartFeeFlag !== '0' && includeStartFeeFlag !== 'false';
368
+ const pricingInfoStartFeeUsd = Number(
369
+ pricingInfoChargeEvents?.['apify-actor-start']?.eventPriceUsd
370
+ );
371
+ const explicitStartFeeUsd =
372
+ env?.ACTOR_START_FEE_USD !== undefined &&
373
+ env?.ACTOR_START_FEE_USD !== null &&
374
+ String(env.ACTOR_START_FEE_USD).trim() !== ''
375
+ ? Number(env.ACTOR_START_FEE_USD)
376
+ : Number.isFinite(pricingInfoStartFeeUsd) && pricingInfoStartFeeUsd >= 0
377
+ ? pricingInfoStartFeeUsd
378
+ : ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
379
+ const startFeeMicros =
380
+ includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
381
+ ? Math.round(explicitStartFeeUsd * 1_000_000)
382
+ : 0;
383
+ const maxBudgetMicros = hasMaxChargeUsd ? Math.round(maxTotalChargeUsdRaw * 1_000_000) : 0;
384
+ let cumulativeChargedInAdapter = 0;
385
+ let adapterChargeLimitLatched = false;
386
+ hooks.actorStartFeeUsdc =
387
+ includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
388
+ ? explicitStartFeeUsd
389
+ : 0;
390
+
391
+ if (kvsId) {
392
+ const checkpointRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${ACTOR_PRICING_CONTRACT.checkpointKey}`;
393
+ const outputRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/OUTPUT`;
394
+
395
+ hooks.loadCheckpoint = async () => {
396
+ const controller = new AbortController();
397
+ const timer = setTimeout(() => controller.abort(), 1500);
398
+ try {
399
+ const res = await fetchImpl(checkpointRecordUrl, {
400
+ method: 'GET',
401
+ headers: {
402
+ authorization: `Bearer ${token}`,
403
+ accept: 'application/json',
404
+ },
405
+ signal: controller.signal,
406
+ });
407
+ if (res.status === 404 || res.status === 204) {
408
+ return null;
409
+ }
410
+ if (res.status >= 200 && res.status < 300) {
411
+ const text = await res.text();
412
+ const loaded = text && text.trim() ? JSON.parse(text) : null;
413
+ if (
414
+ loaded &&
415
+ Array.isArray(loaded.chargedIds) &&
416
+ (!loaded.runId || !runId || String(loaded.runId) === String(runId))
417
+ ) {
418
+ cumulativeChargedInAdapter = loaded.chargedIds.length;
419
+ }
420
+ return loaded;
421
+ }
422
+ throw new Error(`Apify KVS loadCheckpoint failed with HTTP ${res.status}`);
423
+ } finally {
424
+ clearTimeout(timer);
425
+ }
426
+ };
427
+
428
+ hooks.saveCheckpoint = async (snapshot) => {
429
+ const controller = new AbortController();
430
+ const timer = setTimeout(() => controller.abort(), 1500);
431
+ try {
432
+ const res = await fetchImpl(checkpointRecordUrl, {
433
+ method: 'PUT',
434
+ headers: {
435
+ authorization: `Bearer ${token}`,
436
+ 'content-type': 'application/json; charset=utf-8',
437
+ },
438
+ body: JSON.stringify(snapshot),
439
+ signal: controller.signal,
440
+ });
441
+ if (res.status < 200 || res.status >= 300) {
442
+ throw new Error(`Apify KVS saveCheckpoint HTTP ${res.status}`);
443
+ }
444
+ } finally {
445
+ clearTimeout(timer);
446
+ }
447
+ };
448
+
449
+ hooks.saveOutput = async (outputRecord) => {
450
+ const controller = new AbortController();
451
+ const timer = setTimeout(() => controller.abort(), 1500);
452
+ try {
453
+ const res = await fetchImpl(outputRecordUrl, {
454
+ method: 'PUT',
455
+ headers: {
456
+ authorization: `Bearer ${token}`,
457
+ 'content-type': 'application/json; charset=utf-8',
458
+ },
459
+ body: JSON.stringify(outputRecord),
460
+ signal: controller.signal,
461
+ });
462
+ if (res.status < 200 || res.status >= 300) {
463
+ throw new Error(`Apify KVS saveOutput HTTP ${res.status}`);
464
+ }
465
+ } finally {
466
+ clearTimeout(timer);
467
+ }
468
+ };
469
+ }
470
+
471
+ if (datasetId) {
472
+ const datasetItemsUrl = `${apiBase}/v2/datasets/${encodeURIComponent(datasetId)}/items`;
473
+ hooks.listDeliveredIds = async () => {
474
+ const controller = new AbortController();
475
+ const timer = setTimeout(() => controller.abort(), 1500);
476
+ try {
477
+ const res = await fetchImpl(`${datasetItemsUrl}?fields=creativeId&clean=1`, {
478
+ method: 'GET',
479
+ headers: {
480
+ authorization: `Bearer ${token}`,
481
+ accept: 'application/json',
482
+ },
483
+ signal: controller.signal,
484
+ });
485
+ if (res.status >= 200 && res.status < 300) {
486
+ const text = await res.text();
487
+ const parsed = text && text.trim() ? JSON.parse(text) : [];
488
+ const arr = Array.isArray(parsed)
489
+ ? parsed
490
+ : Array.isArray(parsed?.data?.items)
491
+ ? parsed.data.items
492
+ : [];
493
+ return arr
494
+ .map((row) => (typeof row?.creativeId === 'string' ? row.creativeId : null))
495
+ .filter(Boolean);
496
+ }
497
+ throw new Error(`Apify Dataset listDeliveredIds failed with HTTP ${res.status}`);
498
+ } finally {
499
+ clearTimeout(timer);
500
+ }
501
+ };
502
+
503
+ hooks.pushData = async (items) => {
504
+ const controller = new AbortController();
505
+ const timer = setTimeout(() => controller.abort(), 1500);
506
+ try {
507
+ const res = await fetchImpl(datasetItemsUrl, {
508
+ method: 'POST',
509
+ headers: {
510
+ authorization: `Bearer ${token}`,
511
+ 'content-type': 'application/json; charset=utf-8',
512
+ },
513
+ body: JSON.stringify(items),
514
+ signal: controller.signal,
515
+ });
516
+ if (res.status < 200 || res.status >= 300) {
517
+ throw new Error(`Apify Dataset pushData HTTP ${res.status}`);
518
+ }
519
+ } finally {
520
+ clearTimeout(timer);
521
+ }
522
+ };
523
+ }
524
+
525
+ if (runId) {
526
+ const chargeUrl = `${apiBase}/v2/actor-runs/${encodeURIComponent(runId)}/charge`;
527
+ const allowUnbilledAtHome = env?.ADPULSE_ALLOW_UNBILLED_AT_HOME === '1';
528
+ let unbilledOwnerTestLatched = false;
529
+ let latchedPlatformEventName = null;
530
+ const resolvePlatformEventAndPrice = (rawEventName) => {
531
+ const candidate = String(rawEventName || ACTOR_PRICING_CONTRACT.eventName);
532
+ const effectiveName =
533
+ latchedPlatformEventName ||
534
+ (() => {
535
+ if (pricingInfoChargeEvents) {
536
+ if (Object.prototype.hasOwnProperty.call(pricingInfoChargeEvents, candidate)) {
537
+ return candidate;
538
+ }
539
+ if (
540
+ candidate === 'ad-creative-scraped' ||
541
+ candidate === 'ad_transparency_creative_retrieved'
542
+ ) {
543
+ if (
544
+ Object.prototype.hasOwnProperty.call(
545
+ pricingInfoChargeEvents,
546
+ 'ad_transparency_creative_retrieved'
547
+ )
548
+ ) {
549
+ return 'ad_transparency_creative_retrieved';
550
+ }
551
+ if (
552
+ Object.prototype.hasOwnProperty.call(
553
+ pricingInfoChargeEvents,
554
+ 'ad-creative-scraped'
555
+ )
556
+ ) {
557
+ return 'ad-creative-scraped';
558
+ }
559
+ }
560
+ }
561
+ return candidate;
562
+ })();
563
+ const rawEventPrice = Number(pricingInfoChargeEvents?.[effectiveName]?.eventPriceUsd);
564
+ const unitPriceUsdc =
565
+ Number.isFinite(rawEventPrice) && rawEventPrice > 0
566
+ ? rawEventPrice
567
+ : ACTOR_PRICING_CONTRACT.priceUsdc;
568
+ return { eventName: effectiveName, unitPriceUsdc };
569
+ };
570
+ hooks.unitPriceUsdc = resolvePlatformEventAndPrice(
571
+ ACTOR_PRICING_CONTRACT.eventName
572
+ ).unitPriceUsdc;
573
+ hooks.chargeEvent = async (receipt) => {
574
+ const requestedCount = Math.max(0, Number(receipt?.count) || 0);
575
+ const { eventName: primaryEventName, unitPriceUsdc: resolvedUnitPriceUsdc } =
576
+ resolvePlatformEventAndPrice(receipt?.eventName);
577
+ hooks.unitPriceUsdc = resolvedUnitPriceUsdc;
578
+ if (unbilledOwnerTestLatched) {
579
+ return {
580
+ chargedCount: 0,
581
+ eventChargeLimitReached: false,
582
+ unbilledOwnerTestBypass: true,
583
+ chargedCreativeIds: [],
584
+ eventName: primaryEventName,
585
+ unitPriceUsdc: resolvedUnitPriceUsdc,
586
+ };
587
+ }
588
+ if (adapterChargeLimitLatched) {
589
+ return {
590
+ chargedCount: 0,
591
+ eventChargeLimitReached: true,
592
+ chargedCreativeIds: [],
593
+ eventName: primaryEventName,
594
+ unitPriceUsdc: resolvedUnitPriceUsdc,
595
+ };
596
+ }
597
+ const unitPriceMicros = Math.round(resolvedUnitPriceUsdc * 1_000_000);
598
+ if (!Number.isFinite(unitPriceMicros) || unitPriceMicros < 1) {
599
+ throw new Error(
600
+ `Invalid unitPriceUsdc (${String(resolvedUnitPriceUsdc)}): unitPriceMicros must be >= 1`
601
+ );
602
+ }
603
+ const maxPayableEvents = hasMaxChargeUsd
604
+ ? Math.floor(Math.max(0, maxBudgetMicros - startFeeMicros) / unitPriceMicros)
605
+ : Infinity;
606
+ const remainingAllowance = Number.isFinite(maxPayableEvents)
607
+ ? Math.max(0, maxPayableEvents - cumulativeChargedInAdapter)
608
+ : requestedCount;
609
+ const countToCharge = Math.min(requestedCount, remainingAllowance);
610
+ const limitReached =
611
+ Number.isFinite(maxPayableEvents) &&
612
+ cumulativeChargedInAdapter + countToCharge >= maxPayableEvents &&
613
+ countToCharge < requestedCount;
614
+
615
+ if (countToCharge === 0) {
616
+ return {
617
+ chargedCount: 0,
618
+ eventChargeLimitReached: Number.isFinite(maxPayableEvents),
619
+ chargedCreativeIds: [],
620
+ eventName: primaryEventName,
621
+ unitPriceUsdc: resolvedUnitPriceUsdc,
622
+ };
623
+ }
624
+
625
+ const chargeIdempotencyVerified =
626
+ env?.APIFY_CHARGE_IDEMPOTENCY_SUPPORTED === '1' ||
627
+ env?.APIFY_CHARGE_IDEMPOTENCY_SUPPORTED === 'true';
628
+ const maxChargeAttempts = chargeIdempotencyVerified ? 2 : 1;
629
+ const perAttemptTimeoutMs = 900;
630
+ const canStartChargePost = () =>
631
+ !Number.isFinite(receipt?.deadlineAtMs) ||
632
+ Date.now() + perAttemptTimeoutMs + 300 < receipt.deadlineAtMs;
633
+
634
+ const postChargeRequestWithIdempotentRetry = async (
635
+ eventNameToPost,
636
+ idempotencyKeyToPost
637
+ ) => {
638
+ let lastPostErr = null;
639
+ for (let attempt = 1; attempt <= maxChargeAttempts; attempt += 1) {
640
+ if (attempt > 1 && !canStartChargePost()) {
641
+ break;
642
+ }
643
+ const controller = new AbortController();
644
+ const timer = setTimeout(() => controller.abort(), perAttemptTimeoutMs);
645
+ try {
646
+ const res = await fetchImpl(chargeUrl, {
647
+ method: 'POST',
648
+ headers: {
649
+ authorization: `Bearer ${token}`,
650
+ 'content-type': 'application/json; charset=utf-8',
651
+ 'idempotency-key': idempotencyKeyToPost,
652
+ },
653
+ body: JSON.stringify({
654
+ eventName: eventNameToPost,
655
+ count: countToCharge,
656
+ }),
657
+ signal: controller.signal,
658
+ });
659
+ if (
660
+ attempt < maxChargeAttempts &&
661
+ (res.status === 429 || res.status >= 500) &&
662
+ canStartChargePost()
663
+ ) {
664
+ try {
665
+ await res.body?.cancel?.();
666
+ } catch {
667
+ // Ignore body cancel failure on retry
668
+ }
669
+ await new Promise((r) => setTimeout(r, 150));
670
+ continue;
671
+ }
672
+ const bodyText =
673
+ typeof res?.text === 'function'
674
+ ? await Promise.race([
675
+ res.text(),
676
+ new Promise((_, reject) => {
677
+ if (controller.signal.aborted) {
678
+ reject(new Error('AbortError: charge response body timed out'));
679
+ return;
680
+ }
681
+ controller.signal.addEventListener(
682
+ 'abort',
683
+ () => reject(new Error('AbortError: charge response body timed out')),
684
+ { once: true }
685
+ );
686
+ }),
687
+ ])
688
+ : '';
689
+ return { status: res.status, bodyText };
690
+ } catch (err) {
691
+ lastPostErr = err;
692
+ if (attempt < maxChargeAttempts && canStartChargePost()) {
693
+ await new Promise((r) => setTimeout(r, 150));
694
+ continue;
695
+ }
696
+ throw err;
697
+ } finally {
698
+ clearTimeout(timer);
699
+ }
700
+ }
701
+ throw lastPostErr || new Error('Charge POST exceeded run deadline');
702
+ };
703
+
704
+ try {
705
+ let effectiveEventName = primaryEventName;
706
+ let effectiveIdempotencyKey = receipt.idempotencyKey;
707
+ let res = await postChargeRequestWithIdempotentRetry(
708
+ primaryEventName,
709
+ receipt.idempotencyKey
710
+ );
711
+ if (
712
+ res.status === 404 &&
713
+ !pricingInfoChargeEvents &&
714
+ !latchedPlatformEventName &&
715
+ (primaryEventName === 'ad-creative-scraped' ||
716
+ primaryEventName === 'ad_transparency_creative_retrieved')
717
+ ) {
718
+ const aliasEventName =
719
+ primaryEventName === 'ad-creative-scraped'
720
+ ? 'ad_transparency_creative_retrieved'
721
+ : 'ad-creative-scraped';
722
+ const aliasIdempotencyKey = `${receipt.idempotencyKey}-${aliasEventName}`;
723
+ const fallbackRes = await postChargeRequestWithIdempotentRetry(
724
+ aliasEventName,
725
+ aliasIdempotencyKey
726
+ );
727
+ if (fallbackRes.status !== 404) {
728
+ res = fallbackRes;
729
+ effectiveEventName = aliasEventName;
730
+ effectiveIdempotencyKey = aliasIdempotencyKey;
731
+ if (
732
+ (fallbackRes.status >= 200 && fallbackRes.status < 300) ||
733
+ fallbackRes.status === 402
734
+ ) {
735
+ latchedPlatformEventName = aliasEventName;
736
+ }
737
+ }
738
+ }
739
+ if (res.status === 402) {
740
+ // Platform enforced spending ceiling (HTTP 402 Payment Required): latch so no further charge POSTs occur
741
+ adapterChargeLimitLatched = true;
742
+ return {
743
+ chargedCount: 0,
744
+ eventChargeLimitReached: true,
745
+ chargedCreativeIds: [],
746
+ eventName: effectiveEventName,
747
+ unitPriceUsdc: resolvedUnitPriceUsdc,
748
+ idempotencyKey: effectiveIdempotencyKey,
749
+ };
750
+ }
751
+ if (allowUnbilledAtHome && (res.status === 405 || res.status === 403)) {
752
+ let isUnmonetizedNotAllowed = res.status === 405;
753
+ if (res.status === 403) {
754
+ try {
755
+ const errText = res.bodyText || '';
756
+ const errJson = errText && errText.trim() ? JSON.parse(errText) : null;
757
+ isUnmonetizedNotAllowed = errJson?.error?.type === 'operation-not-allowed';
758
+ } catch {
759
+ isUnmonetizedNotAllowed = false;
760
+ }
761
+ }
762
+ if (isUnmonetizedNotAllowed) {
763
+ unbilledOwnerTestLatched = true;
764
+ return {
765
+ chargedCount: 0,
766
+ eventChargeLimitReached: false,
767
+ unbilledOwnerTestBypass: true,
768
+ chargedCreativeIds: [],
769
+ eventName: effectiveEventName,
770
+ unitPriceUsdc: resolvedUnitPriceUsdc,
771
+ idempotencyKey: effectiveIdempotencyKey,
772
+ };
773
+ }
774
+ }
775
+ if (res.status < 200 || res.status >= 300) {
776
+ let errSuffix = '';
777
+ try {
778
+ const errText = res.bodyText || '';
779
+ if (errText && errText.trim()) {
780
+ const errJson = JSON.parse(errText);
781
+ const errType = errJson?.error?.type;
782
+ if (typeof errType === 'string' && /^[A-Za-z0-9_.:-]{1,64}$/.test(errType)) {
783
+ errSuffix = ` (${errType})`;
784
+ }
785
+ }
786
+ } catch {
787
+ errSuffix = '';
788
+ }
789
+ throw new Error(`Apify Pay-per-Event charge HTTP ${res.status}${errSuffix}`);
790
+ }
791
+ const rawText = res.bodyText || '';
792
+ const parsed = rawText && rawText.trim() ? JSON.parse(rawText) : {};
793
+ const dataObj = parsed?.data && typeof parsed.data === 'object' ? parsed.data : parsed;
794
+ // Apify REST `POST /v2/actor-runs/{runId}/charge` returns HTTP 201 Created with `{}` on success.
795
+ // Only fallback to `countToCharge` when `chargedCount` is genuinely omitted; if present but invalid, throw fail-closed.
796
+ const hasExplicitChargedCount =
797
+ Boolean(dataObj) &&
798
+ typeof dataObj === 'object' &&
799
+ Object.prototype.hasOwnProperty.call(dataObj, 'chargedCount') &&
800
+ dataObj.chargedCount !== undefined;
801
+ if (
802
+ hasExplicitChargedCount &&
803
+ (!Number.isInteger(dataObj.chargedCount) ||
804
+ dataObj.chargedCount < 0 ||
805
+ dataObj.chargedCount > countToCharge)
806
+ ) {
807
+ throw new Error(
808
+ `Apify Pay-per-Event charge response contained invalid chargedCount: ${String(dataObj.chargedCount)}`
809
+ );
810
+ }
811
+ const resolvedChargedCount = hasExplicitChargedCount
812
+ ? dataObj.chargedCount
813
+ : countToCharge;
814
+ cumulativeChargedInAdapter += resolvedChargedCount;
815
+ const resolvedLimitReached =
816
+ Boolean(dataObj?.eventChargeLimitReached) ||
817
+ limitReached ||
818
+ (Number.isFinite(maxPayableEvents) && cumulativeChargedInAdapter >= maxPayableEvents);
819
+ let subsetIds;
820
+ if (Array.isArray(dataObj?.chargedCreativeIds)) {
821
+ const reqIds = Array.isArray(receipt?.creativeIds) ? receipt.creativeIds : null;
822
+ const uniqueValidIds = new Set(
823
+ dataObj.chargedCreativeIds.filter(
824
+ (id) => typeof id === 'string' && (!reqIds || reqIds.includes(id))
825
+ )
826
+ );
827
+ if (
828
+ dataObj.chargedCreativeIds.length !== resolvedChargedCount ||
829
+ uniqueValidIds.size !== resolvedChargedCount
830
+ ) {
831
+ throw new Error(
832
+ 'Apify Pay-per-Event charge response contained invalid chargedCreativeIds'
833
+ );
834
+ }
835
+ subsetIds = Array.from(uniqueValidIds);
836
+ } else if (Array.isArray(receipt?.creativeIds)) {
837
+ subsetIds = receipt.creativeIds.slice(0, resolvedChargedCount);
838
+ }
839
+ return {
840
+ chargedCount: resolvedChargedCount,
841
+ eventChargeLimitReached: resolvedLimitReached,
842
+ eventName: effectiveEventName,
843
+ unitPriceUsdc: resolvedUnitPriceUsdc,
844
+ idempotencyKey: effectiveIdempotencyKey,
845
+ ...(subsetIds ? { chargedCreativeIds: subsetIds } : {}),
846
+ };
847
+ } finally {
848
+ // Per-attempt timeout timers are cleared inside postChargeRequestWithIdempotentRetry
849
+ }
850
+ };
851
+ }
852
+
853
+ return hooks;
854
+ }
855
+
856
+ // Local Apify storage directory mode (`APIFY_LOCAL_STORAGE_DIR`)
857
+ const storeName = kvsId || 'default';
858
+ const dsName = datasetId || 'default';
859
+ const kvsDir = path.join(localStorageDir, 'key_value_stores', storeName);
860
+ const dsDir = path.join(localStorageDir, 'datasets', dsName);
861
+ const checkpointPath = path.join(kvsDir, `${ACTOR_PRICING_CONTRACT.checkpointKey}.json`);
862
+ const outputPath = path.join(kvsDir, 'OUTPUT.json');
863
+
864
+ hooks.loadCheckpoint = async () => {
865
+ if (!fs.existsSync(checkpointPath)) return null;
866
+ return JSON.parse(fs.readFileSync(checkpointPath, 'utf8'));
867
+ };
868
+
869
+ hooks.saveCheckpoint = async (snapshot) => {
870
+ fs.mkdirSync(kvsDir, { recursive: true });
871
+ fs.writeFileSync(checkpointPath, JSON.stringify(snapshot, null, 2), 'utf8');
872
+ };
873
+
874
+ hooks.saveOutput = async (outputRecord) => {
875
+ fs.mkdirSync(kvsDir, { recursive: true });
876
+ fs.writeFileSync(outputPath, JSON.stringify(outputRecord, null, 2), 'utf8');
877
+ };
878
+
879
+ hooks.listDeliveredIds = async () => {
880
+ if (!fs.existsSync(dsDir)) return [];
881
+ const existingFiles = fs.readdirSync(dsDir).filter((f) => f.endsWith('.json'));
882
+ const ids = [];
883
+ for (const file of existingFiles) {
884
+ try {
885
+ const parsed = JSON.parse(fs.readFileSync(path.join(dsDir, file), 'utf8'));
886
+ if (typeof parsed?.creativeId === 'string') {
887
+ ids.push(parsed.creativeId);
888
+ }
889
+ } catch {
890
+ // Ignore unreadable partial files
891
+ }
892
+ }
893
+ return ids;
894
+ };
895
+
896
+ hooks.pushData = async (items) => {
897
+ fs.mkdirSync(dsDir, { recursive: true });
898
+ const existingSeqs = fs
899
+ .readdirSync(dsDir)
900
+ .map((f) => {
901
+ const m = /^(\d+)\.json$/.exec(f);
902
+ return m ? Number(m[1]) : 0;
903
+ })
904
+ .filter((n) => Number.isInteger(n) && n > 0);
905
+ let nextSeq = existingSeqs.length > 0 ? Math.max(...existingSeqs) + 1 : 1;
906
+ const list = Array.isArray(items) ? items : [items];
907
+ for (const item of list) {
908
+ const fileName = `${String(nextSeq++).padStart(9, '0')}.json`;
909
+ fs.writeFileSync(path.join(dsDir, fileName), JSON.stringify(item, null, 2), 'utf8');
910
+ }
911
+ };
912
+
913
+ return hooks;
914
+ }
915
+
916
+ /**
917
+ * Normalizes a persisted KVS billing checkpoint object.
918
+ * If the checkpoint belongs to a different `runId`, discards stale state and starts fresh.
919
+ *
920
+ * @param {unknown} rawState
921
+ * @param {string} runId
922
+ * @returns {object}
923
+ */
924
+ function normalizeBillingCheckpointState(rawState, runId) {
925
+ const s = rawState && typeof rawState === 'object' && !Array.isArray(rawState) ? rawState : {};
926
+ if (s.runId && String(s.runId).trim() !== String(runId).trim()) {
927
+ return {
928
+ runId: String(runId || 'default_run'),
929
+ chargedIds: [],
930
+ deliveredIds: [],
931
+ inFlightIds: [],
932
+ inFlightIdempotencyKey: null,
933
+ unknownOutcomeIds: [],
934
+ stagedItemsById: {},
935
+ };
936
+ }
937
+
938
+ const toUniqueIdArray = (arr) =>
939
+ Array.isArray(arr)
940
+ ? Array.from(new Set(arr.map((x) => String(x || '').trim()).filter((x) => CREATIVE_ID_RE.test(x))))
941
+ : [];
942
+
943
+ const stagedItemsById = {};
944
+ if (s.stagedItemsById && typeof s.stagedItemsById === 'object' && !Array.isArray(s.stagedItemsById)) {
945
+ for (const [k, v] of Object.entries(s.stagedItemsById)) {
946
+ if (CREATIVE_ID_RE.test(k) && v && typeof v === 'object') {
947
+ stagedItemsById[k] = v;
948
+ }
949
+ }
950
+ }
951
+
952
+ return {
953
+ runId: String(s.runId || runId || 'default_run'),
954
+ chargedIds: toUniqueIdArray(s.chargedIds),
955
+ deliveredIds: toUniqueIdArray(s.deliveredIds),
956
+ inFlightIds: toUniqueIdArray(s.inFlightIds),
957
+ inFlightIdempotencyKey:
958
+ typeof s.inFlightIdempotencyKey === 'string' && s.inFlightIdempotencyKey.trim()
959
+ ? s.inFlightIdempotencyKey.trim()
960
+ : null,
961
+ unknownOutcomeIds: toUniqueIdArray(s.unknownOutcomeIds),
962
+ stagedItemsById,
963
+ };
964
+ }
965
+
966
+ /**
967
+ * Executes the end-to-end Google Ads Transparency Actor workflow with crash-safe billing
968
+ * checkpointing, strict `ChargeResult` validation, deterministic partial-limit prefix delivery,
969
+ * mid-stream cursor rewind, and fail-closed zero-charge protection.
970
+ *
971
+ * @param {object} input
972
+ * @param {object} [hooks]
973
+ * @param {Function} [hooks.pushData]
974
+ * @param {Function} [hooks.chargeEvent]
975
+ * @param {Function} [hooks.loadCheckpoint]
976
+ * @param {Function} [hooks.saveCheckpoint]
977
+ * @param {Function} [hooks.fetchImpl]
978
+ * @param {Function} [hooks.nowMs]
979
+ * @param {string} [hooks.runId]
980
+ * @returns {Promise<object>}
981
+ */
982
+ export async function runGoogleAdsTransparencyActor(input = {}, hooks = {}) {
983
+ const nowMs = typeof hooks.nowMs === 'function' ? hooks.nowMs : () => Date.now();
984
+ const startedAtMs = nowMs();
985
+ // Authoritative runId precedence: hooks.runId -> ACTOR_RUN_ID -> APIFY_ACTOR_RUN_ID -> input.runId -> 'default_run'
986
+ const runId = String(
987
+ hooks.runId ||
988
+ process.env.ACTOR_RUN_ID ||
989
+ process.env.APIFY_ACTOR_RUN_ID ||
990
+ input?.runId ||
991
+ 'default_run'
992
+ );
993
+ const hasChargeHook = typeof hooks.chargeEvent === 'function';
994
+ let billingMode =
995
+ hasChargeHook || hooks.isAtHome === true ? 'APIFY_PAY_PER_EVENT' : 'LOCAL_UNBILLED';
996
+
997
+ const maxRunDurationMs = Math.min(
998
+ DEFAULT_RUN_DEADLINE_MS,
999
+ Math.max(1000, Number(input?.maxRunDurationMs) || DEFAULT_RUN_DEADLINE_MS)
1000
+ );
1001
+ const deadlineAtMs = startedAtMs + maxRunDurationMs;
1002
+ const billingReserveMs =
1003
+ maxRunDurationMs >= 6000 ? 2500 : Math.min(1000, Math.floor(maxRunDurationMs * 0.25));
1004
+ const rpcDeadlineAtMs = hasChargeHook
1005
+ ? Math.max(startedAtMs + 500, deadlineAtMs - billingReserveMs)
1006
+ : deadlineAtMs;
1007
+ const step4ReserveMs = hasChargeHook
1008
+ ? Math.min(2100, Math.floor(billingReserveMs * 0.85))
1009
+ : 0;
1010
+
1011
+ const finalizeActorReturn = async (finalResult) => {
1012
+ if (typeof hooks.saveOutput === 'function' && nowMs() < deadlineAtMs) {
1013
+ try {
1014
+ await hooks.saveOutput({
1015
+ ok: finalResult.ok,
1016
+ status: finalResult.status,
1017
+ reason: finalResult.reason || null,
1018
+ billingMode: finalResult.billingMode,
1019
+ extractedCount: finalResult.extractedCount,
1020
+ deliveredCount: finalResult.deliveredCount,
1021
+ billableCount: finalResult.billableCount,
1022
+ chargedEventCount: finalResult.chargedEventCount,
1023
+ pagination: finalResult.pagination,
1024
+ ...(Array.isArray(finalResult.warnings) && finalResult.warnings.length > 0
1025
+ ? { warnings: finalResult.warnings }
1026
+ : {}),
1027
+ queryMeta: finalResult.queryMeta,
1028
+ selectedEntities: finalResult.selectedEntities,
1029
+ telemetry: finalResult.telemetry,
1030
+ });
1031
+ } catch {
1032
+ // Non-fatal KVS summary write
1033
+ }
1034
+ }
1035
+ return finalResult;
1036
+ };
1037
+
1038
+ const pushDataWithRetry = async (itemsToPush) => {
1039
+ if (
1040
+ typeof hooks.pushData !== 'function' ||
1041
+ !Array.isArray(itemsToPush) ||
1042
+ itemsToPush.length === 0
1043
+ ) {
1044
+ return { ok: true };
1045
+ }
1046
+ let pendingItems = [...itemsToPush];
1047
+ let lastErr = null;
1048
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
1049
+ try {
1050
+ await hooks.pushData(pendingItems);
1051
+ return { ok: true };
1052
+ } catch (err) {
1053
+ lastErr = err;
1054
+ if (attempt < 2) {
1055
+ if (nowMs() + 1500 >= deadlineAtMs || typeof hooks.listDeliveredIds !== 'function') {
1056
+ return { ok: false, error: lastErr };
1057
+ }
1058
+ try {
1059
+ const committedIds = new Set(await hooks.listDeliveredIds());
1060
+ pendingItems = pendingItems.filter((it) => !committedIds.has(it.creativeId));
1061
+ if (pendingItems.length === 0) {
1062
+ return { ok: true };
1063
+ }
1064
+ } catch (reconcileErr) {
1065
+ return { ok: false, error: reconcileErr || lastErr };
1066
+ }
1067
+ }
1068
+ }
1069
+ }
1070
+ return { ok: false, error: lastErr };
1071
+ };
1072
+
1073
+ // Step 0.0: Fail-Closed At-Home Cloud Hook Completeness Gate
1074
+ // In Apify Cloud (`hooks.isAtHome === true`), require all 4 platform hooks (`chargeEvent`,
1075
+ // `pushData`, `loadCheckpoint`, `saveCheckpoint`) before any RPC fetch or billing occurs.
1076
+ if (
1077
+ hooks.isAtHome === true &&
1078
+ (typeof hooks.chargeEvent !== 'function' ||
1079
+ typeof hooks.pushData !== 'function' ||
1080
+ typeof hooks.loadCheckpoint !== 'function' ||
1081
+ typeof hooks.saveCheckpoint !== 'function')
1082
+ ) {
1083
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1084
+ return finalizeActorReturn({
1085
+ ok: false,
1086
+ status: 'PARTIAL_BILLING_HALT',
1087
+ reason:
1088
+ 'Incomplete Apify Cloud runtime hooks (missing chargeEvent, pushData, loadCheckpoint, or saveCheckpoint in at-home mode)',
1089
+ billingMode,
1090
+ queryMeta: null,
1091
+ selectedEntities: [],
1092
+ items: [],
1093
+ extractedCount: 0,
1094
+ deliveredCount: 0,
1095
+ billableCount: 0,
1096
+ chargedEventCount: 0,
1097
+ unknownOutcomeIds: [],
1098
+ chargeReceipts: [],
1099
+ pagination: {
1100
+ nextPageToken: input?.pageToken || null,
1101
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1102
+ paginationIncompleteReason: 'CLOUD_HOOKS_INCOMPLETE',
1103
+ },
1104
+ telemetry: computeRunEconomicsTelemetry({
1105
+ billableCount: 0,
1106
+ chargedEventCount: 0,
1107
+ durationMs,
1108
+ requestCount: 1,
1109
+ }),
1110
+ });
1111
+ }
1112
+
1113
+ // Step 0: Load & Reconcile Crash-Safe Billing Checkpoint (fail-closed if loadCheckpoint throws)
1114
+ let rawLoadedCheckpoint = null;
1115
+ if (typeof hooks.loadCheckpoint === 'function') {
1116
+ try {
1117
+ rawLoadedCheckpoint = await hooks.loadCheckpoint();
1118
+ } catch (cpErr) {
1119
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1120
+ return finalizeActorReturn({
1121
+ ok: false,
1122
+ status: 'PARTIAL_BILLING_HALT',
1123
+ reason: `Failed to load billing checkpoint fail-closed: ${cpErr?.message || String(cpErr)}`,
1124
+ billingMode,
1125
+ queryMeta: null,
1126
+ selectedEntities: [],
1127
+ items: [],
1128
+ extractedCount: 0,
1129
+ deliveredCount: 0,
1130
+ billableCount: 0,
1131
+ chargedEventCount: 0,
1132
+ unknownOutcomeIds: [],
1133
+ chargeReceipts: [],
1134
+ pagination: {
1135
+ nextPageToken: input?.pageToken || null,
1136
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1137
+ paginationIncompleteReason: 'CHECKPOINT_LOAD_FAILED',
1138
+ },
1139
+ telemetry: computeRunEconomicsTelemetry({
1140
+ billableCount: 0,
1141
+ chargedEventCount: 0,
1142
+ durationMs,
1143
+ requestCount: 1,
1144
+ }),
1145
+ });
1146
+ }
1147
+ }
1148
+ const checkpoint = normalizeBillingCheckpointState(rawLoadedCheckpoint, runId);
1149
+
1150
+ const chargedSet = new Set(checkpoint.chargedIds);
1151
+ const deliveredSet = new Set(checkpoint.deliveredIds);
1152
+ const unknownOutcomeSet = new Set(checkpoint.unknownOutcomeIds);
1153
+ let inFlightSet = new Set(checkpoint.inFlightIds);
1154
+ let inFlightIdempotencyKey = checkpoint.inFlightIdempotencyKey || null;
1155
+ const stagedItemsById = { ...checkpoint.stagedItemsById };
1156
+
1157
+ const persistCheckpoint = async () => {
1158
+ const snapshot = {
1159
+ runId,
1160
+ chargedIds: Array.from(chargedSet),
1161
+ deliveredIds: Array.from(deliveredSet),
1162
+ inFlightIds: Array.from(inFlightSet),
1163
+ inFlightIdempotencyKey: inFlightSet.size > 0 ? inFlightIdempotencyKey : null,
1164
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1165
+ stagedItemsById,
1166
+ };
1167
+ if (typeof hooks.saveCheckpoint === 'function') {
1168
+ await hooks.saveCheckpoint(snapshot);
1169
+ }
1170
+ return snapshot;
1171
+ };
1172
+
1173
+ const reconciledItems = [];
1174
+ let deliveredCount = deliveredSet.size;
1175
+
1176
+ // Step 0a: Reconcile any already-paid items (`chargedIds \ deliveredIds`) without re-charging.
1177
+ // First check `hooks.listDeliveredIds()` if available so items already committed to dataset before a lost ACK
1178
+ // are marked delivered without duplicate dataset POSTs. Fail closed if dataset listing throws.
1179
+ const paidUndeliveredIds = Array.from(chargedSet).filter((id) => !deliveredSet.has(id));
1180
+ if (paidUndeliveredIds.length > 0) {
1181
+ let alreadyInDatasetSet = new Set();
1182
+ if (typeof hooks.listDeliveredIds === 'function') {
1183
+ try {
1184
+ const existingDsIds = await hooks.listDeliveredIds();
1185
+ if (Array.isArray(existingDsIds)) {
1186
+ alreadyInDatasetSet = new Set(existingDsIds);
1187
+ }
1188
+ } catch (dsListErr) {
1189
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1190
+ return finalizeActorReturn({
1191
+ ok: false,
1192
+ status: 'PARTIAL_BILLING_HALT',
1193
+ reason: `Failed to verify delivered dataset IDs during reconciliation: ${dsListErr?.message || String(dsListErr)}`,
1194
+ billingMode,
1195
+ queryMeta: null,
1196
+ selectedEntities: [],
1197
+ items: [],
1198
+ extractedCount: 0,
1199
+ deliveredCount,
1200
+ billableCount: 0,
1201
+ chargedEventCount: chargedSet.size,
1202
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1203
+ chargeReceipts: [],
1204
+ pagination: {
1205
+ nextPageToken: input?.pageToken || null,
1206
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1207
+ paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
1208
+ },
1209
+ telemetry: computeRunEconomicsTelemetry({
1210
+ billableCount: 0,
1211
+ chargedEventCount: chargedSet.size,
1212
+ durationMs,
1213
+ requestCount: 1,
1214
+ }),
1215
+ });
1216
+ }
1217
+ }
1218
+
1219
+ const deliverablePairs = [];
1220
+ for (const id of paidUndeliveredIds) {
1221
+ const staged = stagedItemsById[id];
1222
+ if (alreadyInDatasetSet.has(id)) {
1223
+ deliveredSet.add(id);
1224
+ if (staged && typeof staged === 'object' && staged.creativeId === id) {
1225
+ reconciledItems.push(staged);
1226
+ }
1227
+ delete stagedItemsById[id];
1228
+ } else if (staged && typeof staged === 'object' && staged.creativeId === id) {
1229
+ deliverablePairs.push({ id, item: staged });
1230
+ } else {
1231
+ unknownOutcomeSet.add(id);
1232
+ }
1233
+ }
1234
+
1235
+ if (deliverablePairs.length > 0) {
1236
+ const itemsToDeliver = deliverablePairs.map((p) => p.item);
1237
+ const pushRes = await pushDataWithRetry(itemsToDeliver);
1238
+ if (!pushRes.ok) {
1239
+ await persistCheckpoint();
1240
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1241
+ return finalizeActorReturn({
1242
+ ok: false,
1243
+ status: 'PARTIAL_BILLING_HALT',
1244
+ reason: `Dataset pushData failed during reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
1245
+ billingMode,
1246
+ queryMeta: null,
1247
+ selectedEntities: [],
1248
+ items: reconciledItems,
1249
+ extractedCount: 0,
1250
+ deliveredCount,
1251
+ billableCount: reconciledItems.length,
1252
+ chargedEventCount: chargedSet.size,
1253
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1254
+ chargeReceipts: [],
1255
+ pagination: {
1256
+ nextPageToken: input?.pageToken || null,
1257
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1258
+ paginationIncompleteReason: 'DATASET_PUSH_FAILED',
1259
+ },
1260
+ telemetry: computeRunEconomicsTelemetry({
1261
+ billableCount: reconciledItems.length,
1262
+ chargedEventCount: chargedSet.size,
1263
+ durationMs,
1264
+ requestCount: 1,
1265
+ }),
1266
+ });
1267
+ }
1268
+ for (const { id, item } of deliverablePairs) {
1269
+ deliveredSet.add(id);
1270
+ delete stagedItemsById[id];
1271
+ reconciledItems.push(item);
1272
+ }
1273
+ }
1274
+ deliveredCount = deliveredSet.size;
1275
+ await persistCheckpoint();
1276
+ }
1277
+
1278
+ // Step 0b: Reconcile leftover `inFlightIds` via stored `inFlightIdempotencyKey` when available; otherwise transition to `unknownOutcomeIds`
1279
+ if (inFlightSet.size > 0) {
1280
+ const inFlightArr = Array.from(inFlightSet);
1281
+ const canReplayInFlight =
1282
+ hooks.idempotentChargeRetryEnabled === true &&
1283
+ Boolean(inFlightIdempotencyKey) &&
1284
+ typeof hooks.chargeEvent === 'function' &&
1285
+ inFlightArr.every((id) => stagedItemsById[id] && stagedItemsById[id].creativeId === id);
1286
+
1287
+ let reconciledInFlight = false;
1288
+ if (canReplayInFlight) {
1289
+ try {
1290
+ const replayRes = await hooks.chargeEvent({
1291
+ idempotencyKey: inFlightIdempotencyKey,
1292
+ eventName: ACTOR_PRICING_CONTRACT.eventName,
1293
+ count: inFlightArr.length,
1294
+ unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
1295
+ chunkAmountUsdc: Number(
1296
+ (inFlightArr.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
1297
+ ),
1298
+ creativeIds: inFlightArr,
1299
+ deadlineAtMs,
1300
+ });
1301
+ if (replayRes && replayRes.chargedCount === inFlightArr.length) {
1302
+ for (const id of inFlightArr) {
1303
+ chargedSet.add(id);
1304
+ }
1305
+ inFlightSet = new Set();
1306
+ inFlightIdempotencyKey = null;
1307
+ await persistCheckpoint();
1308
+
1309
+ let alreadyInDatasetSet = new Set();
1310
+ if (typeof hooks.listDeliveredIds === 'function') {
1311
+ try {
1312
+ const existingDsIds = await hooks.listDeliveredIds();
1313
+ if (Array.isArray(existingDsIds)) {
1314
+ alreadyInDatasetSet = new Set(existingDsIds);
1315
+ }
1316
+ } catch (dsListErr) {
1317
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1318
+ return finalizeActorReturn({
1319
+ ok: false,
1320
+ status: 'PARTIAL_BILLING_HALT',
1321
+ reason: `Failed to verify delivered dataset IDs during in-flight replay reconciliation: ${dsListErr?.message || String(dsListErr)}`,
1322
+ billingMode,
1323
+ queryMeta: null,
1324
+ selectedEntities: [],
1325
+ items: reconciledItems,
1326
+ extractedCount: 0,
1327
+ deliveredCount,
1328
+ billableCount: reconciledItems.length,
1329
+ chargedEventCount: chargedSet.size,
1330
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1331
+ chargeReceipts: [],
1332
+ pagination: {
1333
+ nextPageToken: input?.pageToken || null,
1334
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1335
+ paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
1336
+ },
1337
+ telemetry: computeRunEconomicsTelemetry({
1338
+ billableCount: reconciledItems.length,
1339
+ chargedEventCount: chargedSet.size,
1340
+ durationMs,
1341
+ requestCount: 1,
1342
+ }),
1343
+ });
1344
+ }
1345
+ }
1346
+
1347
+ const itemsToDeliver = [];
1348
+ for (const id of inFlightArr) {
1349
+ const staged = stagedItemsById[id];
1350
+ if (alreadyInDatasetSet.has(id)) {
1351
+ deliveredSet.add(id);
1352
+ if (staged) reconciledItems.push(staged);
1353
+ delete stagedItemsById[id];
1354
+ } else if (staged) {
1355
+ itemsToDeliver.push(staged);
1356
+ }
1357
+ }
1358
+
1359
+ if (itemsToDeliver.length > 0) {
1360
+ const pushRes = await pushDataWithRetry(itemsToDeliver);
1361
+ if (!pushRes.ok) {
1362
+ await persistCheckpoint();
1363
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1364
+ return finalizeActorReturn({
1365
+ ok: false,
1366
+ status: 'PARTIAL_BILLING_HALT',
1367
+ reason: `Dataset pushData failed during in-flight replay reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
1368
+ billingMode,
1369
+ queryMeta: null,
1370
+ selectedEntities: [],
1371
+ items: reconciledItems,
1372
+ extractedCount: 0,
1373
+ deliveredCount: deliveredSet.size,
1374
+ billableCount: reconciledItems.length,
1375
+ chargedEventCount: chargedSet.size,
1376
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1377
+ chargeReceipts: [],
1378
+ pagination: {
1379
+ nextPageToken: input?.pageToken || null,
1380
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1381
+ paginationIncompleteReason: 'DATASET_PUSH_FAILED',
1382
+ },
1383
+ telemetry: computeRunEconomicsTelemetry({
1384
+ billableCount: reconciledItems.length,
1385
+ chargedEventCount: chargedSet.size,
1386
+ durationMs,
1387
+ requestCount: 1,
1388
+ }),
1389
+ });
1390
+ }
1391
+ for (const item of itemsToDeliver) {
1392
+ deliveredSet.add(item.creativeId);
1393
+ delete stagedItemsById[item.creativeId];
1394
+ reconciledItems.push(item);
1395
+ }
1396
+ }
1397
+ deliveredCount = deliveredSet.size;
1398
+ await persistCheckpoint();
1399
+ reconciledInFlight = true;
1400
+ }
1401
+ } catch {
1402
+ reconciledInFlight = false;
1403
+ }
1404
+ }
1405
+
1406
+ if (!reconciledInFlight) {
1407
+ for (const id of inFlightArr) {
1408
+ if (!chargedSet.has(id) && !deliveredSet.has(id)) {
1409
+ unknownOutcomeSet.add(id);
1410
+ }
1411
+ }
1412
+ inFlightSet = new Set();
1413
+ inFlightIdempotencyKey = null;
1414
+ await persistCheckpoint();
1415
+ }
1416
+ }
1417
+
1418
+ if (unknownOutcomeSet.size > 0 && input?.strictBillingResume !== false) {
1419
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1420
+ return finalizeActorReturn({
1421
+ ok: false,
1422
+ status: 'PARTIAL_BILLING_HALT',
1423
+ reason: `Halted due to ${unknownOutcomeSet.size} unreconciled creative(s) in UNKNOWN_CHARGE_OUTCOME state`,
1424
+ billingMode,
1425
+ queryMeta: null,
1426
+ selectedEntities: [],
1427
+ items: reconciledItems,
1428
+ extractedCount: 0,
1429
+ deliveredCount,
1430
+ billableCount: reconciledItems.length,
1431
+ chargedEventCount: chargedSet.size,
1432
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1433
+ chargeReceipts: [],
1434
+ pagination: {
1435
+ nextPageToken: input?.pageToken || null,
1436
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1437
+ paginationIncompleteReason: 'UNKNOWN_CHARGE_OUTCOME',
1438
+ },
1439
+ telemetry: computeRunEconomicsTelemetry({
1440
+ billableCount: reconciledItems.length,
1441
+ chargedEventCount: chargedSet.size,
1442
+ durationMs,
1443
+ requestCount: 1,
1444
+ }),
1445
+ });
1446
+ }
1447
+
1448
+ const sleepFn =
1449
+ typeof hooks.sleepFn === 'function'
1450
+ ? hooks.sleepFn
1451
+ : typeof input?.sleepFn === 'function'
1452
+ ? input.sleepFn
1453
+ : undefined;
1454
+ const randomFn =
1455
+ typeof hooks.randomFn === 'function'
1456
+ ? hooks.randomFn
1457
+ : typeof input?.randomFn === 'function'
1458
+ ? input.randomFn
1459
+ : undefined;
1460
+
1461
+ // Step 1: Resolve Advertiser Entities
1462
+ const entityRes = await resolveAdvertiserEntities({
1463
+ query: input?.query,
1464
+ advertiserId: input?.advertiserId,
1465
+ advertiserName: input?.advertiserName,
1466
+ countryCode: input?.countryCode,
1467
+ includeSubsidiaries: input?.includeSubsidiaries,
1468
+ maxEntities: input?.maxEntities,
1469
+ fetchImpl: hooks.fetchImpl,
1470
+ nowFn: nowMs,
1471
+ sleepFn,
1472
+ randomFn,
1473
+ timeoutMs: input?.timeoutMs,
1474
+ maxRetries: input?.maxRetries,
1475
+ deadlineAtMs: rpcDeadlineAtMs,
1476
+ });
1477
+
1478
+ if (!entityRes.ok) {
1479
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1480
+ const isPreChargeDeadline =
1481
+ hasChargeHook &&
1482
+ nowMs() >= rpcDeadlineAtMs &&
1483
+ /deadline exceeded/i.test(String(entityRes.reason || ''));
1484
+ return finalizeActorReturn({
1485
+ ok: false,
1486
+ status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : entityRes.status,
1487
+ reason: entityRes.reason,
1488
+ billingMode,
1489
+ queryMeta: entityRes.queryMeta,
1490
+ selectedEntities: [],
1491
+ items: reconciledItems,
1492
+ extractedCount: 0,
1493
+ deliveredCount,
1494
+ billableCount: reconciledItems.length,
1495
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1496
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1497
+ chargeReceipts: [],
1498
+ pagination: {
1499
+ nextPageToken: input?.pageToken || null,
1500
+ paginationStatus: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_HAS_MORE' : entityRes.status,
1501
+ paginationIncompleteReason: isPreChargeDeadline
1502
+ ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
1503
+ : entityRes.reason || null,
1504
+ },
1505
+ telemetry: computeRunEconomicsTelemetry({
1506
+ billableCount: reconciledItems.length,
1507
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1508
+ durationMs,
1509
+ requestCount: Math.max(1, entityRes.rpcAttempts),
1510
+ }),
1511
+ });
1512
+ }
1513
+
1514
+ if (entityRes.status === 'EMPTY_RESULT' || entityRes.selectedEntities.length === 0) {
1515
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1516
+ return finalizeActorReturn({
1517
+ ok: true,
1518
+ status: 'EMPTY_RESULT',
1519
+ billingMode,
1520
+ queryMeta: entityRes.queryMeta,
1521
+ selectedEntities: [],
1522
+ items: reconciledItems,
1523
+ extractedCount: 0,
1524
+ deliveredCount,
1525
+ billableCount: reconciledItems.length,
1526
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1527
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1528
+ chargeReceipts: [],
1529
+ pagination: {
1530
+ nextPageToken: null,
1531
+ paginationStatus: 'EXHAUSTED',
1532
+ paginationIncompleteReason: null,
1533
+ },
1534
+ telemetry: computeRunEconomicsTelemetry({
1535
+ billableCount: reconciledItems.length,
1536
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1537
+ durationMs,
1538
+ requestCount: Math.max(1, entityRes.rpcAttempts),
1539
+ }),
1540
+ });
1541
+ }
1542
+
1543
+ // Step 2: Fetch Creatives across selected verified advertiser entities
1544
+ const advertiserIds = entityRes.selectedEntities.map((e) => e.advertiserId);
1545
+ const creativesRes = await fetchAdvertiserCreatives({
1546
+ advertiserIds,
1547
+ entities: entityRes.selectedEntities,
1548
+ maxResults: input?.maxResults,
1549
+ maxPages: input?.maxPages,
1550
+ pageSize: input?.pageSize,
1551
+ pageToken: input?.pageToken,
1552
+ countryCode: input?.countryCode,
1553
+ formatFilter: input?.formatFilter,
1554
+ fetchImpl: hooks.fetchImpl,
1555
+ nowFn: nowMs,
1556
+ sleepFn,
1557
+ randomFn,
1558
+ timeoutMs: input?.timeoutMs,
1559
+ maxRetries: input?.maxRetries,
1560
+ deadlineAtMs: rpcDeadlineAtMs,
1561
+ });
1562
+
1563
+ let totalRpcRequests = entityRes.rpcAttempts + creativesRes.rpcAttempts;
1564
+
1565
+ if (!creativesRes.ok) {
1566
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1567
+ const isPreChargeDeadline =
1568
+ hasChargeHook &&
1569
+ nowMs() >= rpcDeadlineAtMs &&
1570
+ /deadline exceeded/i.test(String(creativesRes.reason || ''));
1571
+ return finalizeActorReturn({
1572
+ ok: false,
1573
+ status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : creativesRes.status,
1574
+ reason: creativesRes.reason,
1575
+ billingMode,
1576
+ queryMeta: entityRes.queryMeta,
1577
+ selectedEntities: entityRes.selectedEntities,
1578
+ items: reconciledItems,
1579
+ extractedCount: 0,
1580
+ deliveredCount,
1581
+ billableCount: reconciledItems.length,
1582
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1583
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1584
+ chargeReceipts: [],
1585
+ pagination: {
1586
+ nextPageToken: input?.pageToken || null,
1587
+ paginationStatus: isPreChargeDeadline
1588
+ ? 'DEADLINE_EXCEEDED_HAS_MORE'
1589
+ : creativesRes.paginationStatus || creativesRes.status,
1590
+ paginationIncompleteReason: isPreChargeDeadline
1591
+ ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
1592
+ : creativesRes.paginationIncompleteReason || creativesRes.reason || null,
1593
+ },
1594
+ telemetry: computeRunEconomicsTelemetry({
1595
+ billableCount: reconciledItems.length,
1596
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1597
+ durationMs,
1598
+ requestCount: Math.max(1, totalRpcRequests),
1599
+ }),
1600
+ });
1601
+ }
1602
+
1603
+ const maxResultsCap = Math.min(200, Math.max(1, Number(input?.maxResults) || 40));
1604
+
1605
+ // Step 3: Strict Deduplication & Validation Gate before emitting Pay-per-Event charges
1606
+ const uniqueExtractedItems = [];
1607
+ const seenInRunIds = new Set();
1608
+
1609
+ for (const item of creativesRes.creatives) {
1610
+ if (
1611
+ !item ||
1612
+ !ADVERTISER_ID_RE.test(String(item.advertiserId || '')) ||
1613
+ !CREATIVE_ID_RE.test(String(item.creativeId || ''))
1614
+ ) {
1615
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1616
+ return finalizeActorReturn({
1617
+ ok: false,
1618
+ status: 'SCHEMA_DRIFT_HALT',
1619
+ reason: 'Normalized creative record failed AR.../CR... identifier gate prior to billing',
1620
+ billingMode,
1621
+ queryMeta: entityRes.queryMeta,
1622
+ selectedEntities: entityRes.selectedEntities,
1623
+ items: reconciledItems,
1624
+ extractedCount: 0,
1625
+ deliveredCount,
1626
+ billableCount: reconciledItems.length,
1627
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1628
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1629
+ chargeReceipts: [],
1630
+ pagination: {
1631
+ nextPageToken: null,
1632
+ paginationStatus: 'SCHEMA_DRIFT_HALT',
1633
+ paginationIncompleteReason: 'INVALID_CREATIVE_IDENTIFIER',
1634
+ },
1635
+ telemetry: computeRunEconomicsTelemetry({
1636
+ billableCount: reconciledItems.length,
1637
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1638
+ durationMs,
1639
+ requestCount: Math.max(1, totalRpcRequests),
1640
+ }),
1641
+ });
1642
+ }
1643
+
1644
+ if (!seenInRunIds.has(item.creativeId)) {
1645
+ seenInRunIds.add(item.creativeId);
1646
+ uniqueExtractedItems.push(item);
1647
+ if (uniqueExtractedItems.length >= maxResultsCap) {
1648
+ break;
1649
+ }
1650
+ }
1651
+ }
1652
+
1653
+ // Optional Step 3b: Enrich up to 10 creatives (default 5) via LookupService/GetCreativeById when includeCreativeDetails === true
1654
+ if (input?.includeCreativeDetails === true && uniqueExtractedItems.length > 0) {
1655
+ const requestedEnrichCap = Number(input?.maxEnrichItems ?? 5);
1656
+ const enrichCap = Math.min(
1657
+ 10,
1658
+ Math.max(1, Number.isFinite(requestedEnrichCap) ? Math.floor(requestedEnrichCap) : 5),
1659
+ uniqueExtractedItems.length
1660
+ );
1661
+ let rateLimitedInEnrich = false;
1662
+ for (let i = 0; i < enrichCap; i += 1) {
1663
+ const targetItem = uniqueExtractedItems[i];
1664
+ if (rateLimitedInEnrich) {
1665
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1666
+ 'NOT_ATTEMPTED_RATE_LIMITED'
1667
+ );
1668
+ continue;
1669
+ }
1670
+ if (nowMs() >= rpcDeadlineAtMs) {
1671
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1672
+ 'NOT_ATTEMPTED_DEADLINE'
1673
+ );
1674
+ continue;
1675
+ }
1676
+ const detailRes = await getCreativeDetailsById({
1677
+ advertiserId: targetItem.advertiserId,
1678
+ creativeId: targetItem.creativeId,
1679
+ includeAdCopy: Boolean(input?.includeAdCopy),
1680
+ fallbackPreviewUrl: targetItem.previewUrl,
1681
+ fetchImpl: hooks.fetchImpl,
1682
+ nowFn: nowMs,
1683
+ sleepFn,
1684
+ randomFn,
1685
+ timeoutMs: input?.timeoutMs,
1686
+ maxRetries: input?.maxRetries !== undefined ? input.maxRetries : 1,
1687
+ deadlineAtMs: rpcDeadlineAtMs,
1688
+ });
1689
+ totalRpcRequests += detailRes.rpcAttempts;
1690
+ if (detailRes.status === 'UPSTREAM_RATE_LIMITED') {
1691
+ rateLimitedInEnrich = true;
1692
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1693
+ 'NOT_ATTEMPTED_RATE_LIMITED'
1694
+ );
1695
+ continue;
1696
+ }
1697
+ if (detailRes.ok && detailRes.detail) {
1698
+ targetItem.creativeDetail = detailRes.detail;
1699
+ if (detailRes.detail.untrustedAdContent) {
1700
+ targetItem.untrustedAdContent = detailRes.detail.untrustedAdContent;
1701
+ if (
1702
+ detailRes.detail.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
1703
+ detailRes.detail.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
1704
+ ) {
1705
+ rateLimitedInEnrich = true;
1706
+ }
1707
+ }
1708
+ } else if (detailRes.ok && detailRes.untrustedAdContent) {
1709
+ targetItem.untrustedAdContent = detailRes.untrustedAdContent;
1710
+ if (
1711
+ detailRes.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
1712
+ detailRes.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
1713
+ ) {
1714
+ rateLimitedInEnrich = true;
1715
+ }
1716
+ }
1717
+ }
1718
+ }
1719
+
1720
+ const extractedCount = uniqueExtractedItems.length;
1721
+
1722
+ // Filter out items already charged, delivered, or marked unknown in prior checkpoint
1723
+ const newBillableCandidates = uniqueExtractedItems.filter(
1724
+ (item) =>
1725
+ !chargedSet.has(item.creativeId) &&
1726
+ !deliveredSet.has(item.creativeId) &&
1727
+ !unknownOutcomeSet.has(item.creativeId)
1728
+ );
1729
+
1730
+ if (newBillableCandidates.length === 0 && reconciledItems.length === 0) {
1731
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1732
+ return finalizeActorReturn({
1733
+ ok: true,
1734
+ status: 'EMPTY_RESULT',
1735
+ billingMode,
1736
+ queryMeta: entityRes.queryMeta,
1737
+ selectedEntities: entityRes.selectedEntities,
1738
+ items: [],
1739
+ extractedCount,
1740
+ deliveredCount,
1741
+ billableCount: 0,
1742
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1743
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1744
+ chargeReceipts: [],
1745
+ pagination: {
1746
+ nextPageToken: creativesRes.nextPageToken,
1747
+ paginationStatus: creativesRes.paginationStatus,
1748
+ paginationIncompleteReason: creativesRes.paginationIncompleteReason,
1749
+ pagesFetched: creativesRes.pagesFetched,
1750
+ },
1751
+ cursorResumeContext: creativesRes.cursorResumeContext,
1752
+ telemetry: computeRunEconomicsTelemetry({
1753
+ billableCount: 0,
1754
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1755
+ durationMs,
1756
+ requestCount: Math.max(1, totalRpcRequests),
1757
+ unitPriceUsdc: hooks?.unitPriceUsdc ?? ACTOR_PRICING_CONTRACT.priceUsdc,
1758
+ actorStartFeeUsdc: hooks?.actorStartFeeUsdc,
1759
+ }),
1760
+ });
1761
+ }
1762
+
1763
+ // Step 4: Chunked Dataset Push (25 items/chunk) & Crash-Safe Event Charging
1764
+ const chunkSize = ACTOR_PRICING_CONTRACT.chunkSize;
1765
+ const chargeReceipts = [];
1766
+ const newlyDeliveredItems = [];
1767
+ let chargedEventCount = hasChargeHook ? chargedSet.size : 0;
1768
+ let finalStatus = 'OK';
1769
+ let finalOk = true;
1770
+ let haltReason = undefined;
1771
+ const runtimeWarnings = Array.isArray(creativesRes.warnings) ? [...creativesRes.warnings] : [];
1772
+ const persistCheckpointPostCharge = async () => {
1773
+ try {
1774
+ await persistCheckpoint();
1775
+ } catch (cpErr) {
1776
+ runtimeWarnings.push(
1777
+ `Post-charge checkpoint write warning: ${cpErr?.message || String(cpErr)}`
1778
+ );
1779
+ }
1780
+ };
1781
+
1782
+ let lastResolvedUnitPriceUsdc = hooks?.unitPriceUsdc ?? ACTOR_PRICING_CONTRACT.priceUsdc;
1783
+ for (let offset = 0; offset < newBillableCandidates.length; offset += chunkSize) {
1784
+ const effectiveChunkDeadlineMs = hasChargeHook
1785
+ ? deadlineAtMs - step4ReserveMs
1786
+ : deadlineAtMs;
1787
+ if (nowMs() >= effectiveChunkDeadlineMs) {
1788
+ finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
1789
+ finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
1790
+ haltReason = 'Run deadline reserve reached prior to charging next chunk; halted cleanly before billing';
1791
+ break;
1792
+ }
1793
+
1794
+ const unchargedChunk = newBillableCandidates
1795
+ .slice(offset, offset + chunkSize)
1796
+ .filter(
1797
+ (c) =>
1798
+ !chargedSet.has(c.creativeId) &&
1799
+ !deliveredSet.has(c.creativeId) &&
1800
+ !unknownOutcomeSet.has(c.creativeId)
1801
+ );
1802
+
1803
+ if (unchargedChunk.length === 0) {
1804
+ continue;
1805
+ }
1806
+
1807
+ const chunkIds = unchargedChunk.map((c) => c.creativeId);
1808
+ const idempotencyKey = computeChunkIdempotencyKey(runId, chunkIds);
1809
+
1810
+ if (!hasChargeHook) {
1811
+ // Local / MCP unbilled execution mode: deliver items with chargedEventCount = 0
1812
+ const pushRes = await pushDataWithRetry(unchargedChunk);
1813
+ if (!pushRes.ok) {
1814
+ await persistCheckpointPostCharge();
1815
+ finalOk = false;
1816
+ finalStatus = 'PARTIAL_BILLING_HALT';
1817
+ haltReason = `Dataset pushData failed in local mode: ${pushRes.error?.message || String(pushRes.error)}`;
1818
+ break;
1819
+ }
1820
+ for (const c of unchargedChunk) {
1821
+ deliveredSet.add(c.creativeId);
1822
+ delete stagedItemsById[c.creativeId];
1823
+ newlyDeliveredItems.push(c);
1824
+ }
1825
+ deliveredCount = deliveredSet.size;
1826
+ await persistCheckpointPostCharge();
1827
+ continue;
1828
+ }
1829
+
1830
+ // Step 4.1: Persist inFlightIds, inFlightIdempotencyKey, and stagedItemsById BEFORE invoking chargeEvent
1831
+ inFlightSet = new Set(chunkIds);
1832
+ inFlightIdempotencyKey = idempotencyKey;
1833
+ for (const c of unchargedChunk) {
1834
+ stagedItemsById[c.creativeId] = c;
1835
+ }
1836
+ try {
1837
+ await persistCheckpoint();
1838
+ } catch (preCpErr) {
1839
+ inFlightSet = new Set();
1840
+ inFlightIdempotencyKey = null;
1841
+ finalOk = false;
1842
+ finalStatus = 'PARTIAL_BILLING_HALT';
1843
+ haltReason = `Pre-charge checkpoint write failed before billing: ${preCpErr?.message || String(preCpErr)}`;
1844
+ break;
1845
+ }
1846
+
1847
+ if (nowMs() >= effectiveChunkDeadlineMs) {
1848
+ inFlightSet = new Set();
1849
+ inFlightIdempotencyKey = null;
1850
+ await persistCheckpointPostCharge();
1851
+ finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
1852
+ finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
1853
+ haltReason =
1854
+ 'Run deadline reserve reached immediately after pre-charge checkpoint write; cleared inFlightIds and halted before billing';
1855
+ break;
1856
+ }
1857
+
1858
+ const receipt = {
1859
+ idempotencyKey,
1860
+ eventName: ACTOR_PRICING_CONTRACT.eventName,
1861
+ count: unchargedChunk.length,
1862
+ unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
1863
+ chunkAmountUsdc: Number(
1864
+ (unchargedChunk.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
1865
+ ),
1866
+ creativeIds: chunkIds,
1867
+ deadlineAtMs,
1868
+ };
1869
+
1870
+ let chargeResult;
1871
+ try {
1872
+ chargeResult = await hooks.chargeEvent(receipt);
1873
+ } catch (chargeErr) {
1874
+ for (const id of chunkIds) {
1875
+ unknownOutcomeSet.add(id);
1876
+ }
1877
+ inFlightSet = new Set();
1878
+ inFlightIdempotencyKey = null;
1879
+ await persistCheckpointPostCharge();
1880
+ finalOk = false;
1881
+ finalStatus = 'PARTIAL_BILLING_HALT';
1882
+ haltReason = `chargeEvent threw error: ${chargeErr?.message || String(chargeErr)}`;
1883
+ break;
1884
+ }
1885
+
1886
+ // Step 4.2: Strict ChargeResult validation (never fallback to ?? unchargedChunk.length)
1887
+ const chargedCount = chargeResult?.chargedCount;
1888
+ const eventChargeLimitReached = Boolean(chargeResult?.eventChargeLimitReached);
1889
+ const resolvedReceiptEventName =
1890
+ typeof chargeResult?.eventName === 'string' && chargeResult.eventName
1891
+ ? chargeResult.eventName
1892
+ : receipt.eventName;
1893
+ const resolvedReceiptUnitPriceUsdc =
1894
+ Number.isFinite(Number(chargeResult?.unitPriceUsdc)) && Number(chargeResult.unitPriceUsdc) > 0
1895
+ ? Number(chargeResult.unitPriceUsdc)
1896
+ : receipt.unitPriceUsdc;
1897
+ lastResolvedUnitPriceUsdc = resolvedReceiptUnitPriceUsdc;
1898
+ const resolvedReceiptIdempotencyKey =
1899
+ typeof chargeResult?.idempotencyKey === 'string' && chargeResult.idempotencyKey
1900
+ ? chargeResult.idempotencyKey
1901
+ : receipt.idempotencyKey;
1902
+ const buildFinalReceipt = (actualChargedCount, extra = {}) => ({
1903
+ ...receipt,
1904
+ idempotencyKey: resolvedReceiptIdempotencyKey,
1905
+ eventName: resolvedReceiptEventName,
1906
+ unitPriceUsdc: resolvedReceiptUnitPriceUsdc,
1907
+ chunkAmountUsdc: Number((actualChargedCount * resolvedReceiptUnitPriceUsdc).toFixed(6)),
1908
+ chargedCount: actualChargedCount,
1909
+ ...extra,
1910
+ });
1911
+
1912
+ if (
1913
+ !chargeResult ||
1914
+ typeof chargeResult !== 'object' ||
1915
+ !Number.isInteger(chargedCount) ||
1916
+ chargedCount < 0 ||
1917
+ chargedCount > unchargedChunk.length
1918
+ ) {
1919
+ for (const id of chunkIds) {
1920
+ unknownOutcomeSet.add(id);
1921
+ }
1922
+ inFlightSet = new Set();
1923
+ await persistCheckpointPostCharge();
1924
+ finalOk = false;
1925
+ finalStatus = 'PARTIAL_BILLING_HALT';
1926
+ haltReason =
1927
+ 'chargeEvent returned invalid or missing integer chargedCount (transitioned chunk to UNKNOWN_CHARGE_OUTCOME)';
1928
+ break;
1929
+ }
1930
+
1931
+ // Step 4.3: Branch on validated chargedCount
1932
+ if (chargedCount === unchargedChunk.length) {
1933
+ // Full chunk charged: persist chargedIds immediately BEFORE pushData (best-effort so KVS glitch never blocks delivery)
1934
+ for (const id of chunkIds) {
1935
+ chargedSet.add(id);
1936
+ }
1937
+ inFlightSet = new Set();
1938
+ chargedEventCount = chargedSet.size;
1939
+ await persistCheckpointPostCharge();
1940
+
1941
+ const pushRes = await pushDataWithRetry(unchargedChunk);
1942
+ if (!pushRes.ok) {
1943
+ chargeReceipts.push(buildFinalReceipt(chargedCount, { eventChargeLimitReached }));
1944
+ finalOk = false;
1945
+ finalStatus = 'PARTIAL_BILLING_HALT';
1946
+ haltReason = `Dataset pushData failed after charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
1947
+ break;
1948
+ }
1949
+ for (const c of unchargedChunk) {
1950
+ deliveredSet.add(c.creativeId);
1951
+ delete stagedItemsById[c.creativeId];
1952
+ newlyDeliveredItems.push(c);
1953
+ }
1954
+ deliveredCount = deliveredSet.size;
1955
+ await persistCheckpointPostCharge();
1956
+
1957
+ chargeReceipts.push(buildFinalReceipt(chargedCount, { eventChargeLimitReached }));
1958
+
1959
+ if (eventChargeLimitReached) {
1960
+ finalStatus = 'CHARGE_LIMIT_REACHED';
1961
+ break;
1962
+ }
1963
+ } else if (chargedCount === 0) {
1964
+ inFlightSet = new Set();
1965
+ if (chargeResult?.unbilledOwnerTestBypass === true) {
1966
+ billingMode = 'UNBILLED_OWNER_TEST';
1967
+ await persistCheckpointPostCharge();
1968
+ const pushRes = await pushDataWithRetry(unchargedChunk);
1969
+ if (!pushRes.ok) {
1970
+ finalOk = false;
1971
+ finalStatus = 'PARTIAL_BILLING_HALT';
1972
+ haltReason = `Dataset pushData failed in UNBILLED_OWNER_TEST mode: ${pushRes.error?.message || String(pushRes.error)}`;
1973
+ break;
1974
+ }
1975
+ for (const c of unchargedChunk) {
1976
+ deliveredSet.add(c.creativeId);
1977
+ delete stagedItemsById[c.creativeId];
1978
+ newlyDeliveredItems.push(c);
1979
+ }
1980
+ deliveredCount = deliveredSet.size;
1981
+ await persistCheckpointPostCharge();
1982
+ chargeReceipts.push(
1983
+ buildFinalReceipt(0, {
1984
+ eventChargeLimitReached: false,
1985
+ unbilledOwnerTestBypass: true,
1986
+ })
1987
+ );
1988
+ continue;
1989
+ }
1990
+ await persistCheckpointPostCharge();
1991
+ chargeReceipts.push(buildFinalReceipt(0, { eventChargeLimitReached }));
1992
+ if (eventChargeLimitReached) {
1993
+ finalOk = true;
1994
+ finalStatus = 'CHARGE_LIMIT_REACHED';
1995
+ } else {
1996
+ finalOk = false;
1997
+ finalStatus = 'PARTIAL_BILLING_HALT';
1998
+ haltReason = 'chargeEvent returned chargedCount = 0 without eventChargeLimitReached';
1999
+ }
2000
+ break;
2001
+ } else {
2002
+ // Partial chunk charge (0 < chargedCount < unchargedChunk.length)
2003
+ const hasExplicitIdsArray = Array.isArray(chargeResult.chargedCreativeIds);
2004
+ const explicitChargedIds = hasExplicitIdsArray
2005
+ ? Array.from(
2006
+ new Set(chargeResult.chargedCreativeIds.filter((id) => chunkIds.includes(id)))
2007
+ )
2008
+ : null;
2009
+ const isValidExplicitChargedIds = Boolean(
2010
+ explicitChargedIds &&
2011
+ explicitChargedIds.length === chargedCount &&
2012
+ chargeResult.chargedCreativeIds.length === chargedCount
2013
+ );
2014
+
2015
+ if (isValidExplicitChargedIds) {
2016
+ for (const id of explicitChargedIds) {
2017
+ chargedSet.add(id);
2018
+ }
2019
+ inFlightSet = new Set();
2020
+ chargedEventCount = chargedSet.size;
2021
+ await persistCheckpointPostCharge();
2022
+
2023
+ const paidSubset = unchargedChunk.filter((c) => explicitChargedIds.includes(c.creativeId));
2024
+ const pushRes = await pushDataWithRetry(paidSubset);
2025
+ if (!pushRes.ok) {
2026
+ chargeReceipts.push(
2027
+ buildFinalReceipt(chargedCount, {
2028
+ chargedCreativeIds: explicitChargedIds,
2029
+ eventChargeLimitReached,
2030
+ })
2031
+ );
2032
+ finalOk = false;
2033
+ finalStatus = 'PARTIAL_BILLING_HALT';
2034
+ haltReason = `Dataset pushData failed after partial charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
2035
+ break;
2036
+ }
2037
+ for (const c of paidSubset) {
2038
+ deliveredSet.add(c.creativeId);
2039
+ delete stagedItemsById[c.creativeId];
2040
+ newlyDeliveredItems.push(c);
2041
+ }
2042
+ deliveredCount = deliveredSet.size;
2043
+ await persistCheckpointPostCharge();
2044
+
2045
+ chargeReceipts.push(
2046
+ buildFinalReceipt(chargedCount, {
2047
+ chargedCreativeIds: explicitChargedIds,
2048
+ eventChargeLimitReached,
2049
+ })
2050
+ );
2051
+
2052
+ finalOk = eventChargeLimitReached;
2053
+ finalStatus = eventChargeLimitReached ? 'CHARGE_LIMIT_REACHED' : 'PARTIAL_BILLING_HALT';
2054
+ if (!eventChargeLimitReached) {
2055
+ haltReason = `Partial chunk charge (${chargedCount}/${unchargedChunk.length}) without charge limit flag`;
2056
+ }
2057
+ break;
2058
+ } else if (!hasExplicitIdsArray && eventChargeLimitReached) {
2059
+ // Standard Apify SDK ChargeResult when hitting user's max charge ceiling:
2060
+ // Attribute charge deterministically to the ordered prefix `unchargedChunk.slice(0, chargedCount)`
2061
+ // so the customer receives every item they paid for, and leave the remaining tail uncharged.
2062
+ const paidPrefix = unchargedChunk.slice(0, chargedCount);
2063
+ const paidPrefixIds = paidPrefix.map((c) => c.creativeId);
2064
+ for (const id of paidPrefixIds) {
2065
+ chargedSet.add(id);
2066
+ }
2067
+ inFlightSet = new Set();
2068
+ chargedEventCount = chargedSet.size;
2069
+ await persistCheckpointPostCharge();
2070
+
2071
+ const pushRes = await pushDataWithRetry(paidPrefix);
2072
+ if (!pushRes.ok) {
2073
+ chargeReceipts.push(
2074
+ buildFinalReceipt(chargedCount, {
2075
+ chargedCreativeIds: paidPrefixIds,
2076
+ eventChargeLimitReached: true,
2077
+ })
2078
+ );
2079
+ finalOk = false;
2080
+ finalStatus = 'PARTIAL_BILLING_HALT';
2081
+ haltReason = `Dataset pushData failed after charge limit prefix (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
2082
+ break;
2083
+ }
2084
+ for (const c of paidPrefix) {
2085
+ deliveredSet.add(c.creativeId);
2086
+ delete stagedItemsById[c.creativeId];
2087
+ newlyDeliveredItems.push(c);
2088
+ }
2089
+ deliveredCount = deliveredSet.size;
2090
+ await persistCheckpointPostCharge();
2091
+
2092
+ chargeReceipts.push(
2093
+ buildFinalReceipt(chargedCount, {
2094
+ chargedCreativeIds: paidPrefixIds,
2095
+ eventChargeLimitReached: true,
2096
+ })
2097
+ );
2098
+
2099
+ finalOk = true;
2100
+ finalStatus = 'CHARGE_LIMIT_REACHED';
2101
+ break;
2102
+ } else {
2103
+ // Unexpected partial count without eventChargeLimitReached and without explicit IDs ->
2104
+ // move chunk to unknownOutcomeIds and halt fail-closed
2105
+ for (const id of chunkIds) {
2106
+ unknownOutcomeSet.add(id);
2107
+ }
2108
+ inFlightSet = new Set();
2109
+ chargedEventCount = chargedSet.size + chargedCount;
2110
+ await persistCheckpointPostCharge();
2111
+
2112
+ chargeReceipts.push(
2113
+ buildFinalReceipt(chargedCount, {
2114
+ eventChargeLimitReached: false,
2115
+ })
2116
+ );
2117
+
2118
+ finalOk = false;
2119
+ finalStatus = 'PARTIAL_BILLING_HALT';
2120
+ haltReason = `Ambiguous partial chunk charge (${chargedCount}/${unchargedChunk.length}) without chargedCreativeIds attribution or eventChargeLimitReached`;
2121
+ break;
2122
+ }
2123
+ }
2124
+ }
2125
+
2126
+ // Step 5: Rewind pagination cursor if billing halted or reached charge limit before delivering all extracted items
2127
+ let effectiveNextPageToken = creativesRes.nextPageToken;
2128
+ let effectivePaginationStatus = creativesRes.paginationStatus;
2129
+ let effectiveIncompleteReason = creativesRes.paginationIncompleteReason;
2130
+ const undeliveredCount = Math.max(0, newBillableCandidates.length - newlyDeliveredItems.length);
2131
+
2132
+ if (undeliveredCount > 0) {
2133
+ const firstUndeliveredItem = newBillableCandidates.find((c) => !deliveredSet.has(c.creativeId));
2134
+ const firstUndeliveredExtractedIdx = Array.isArray(creativesRes.creatives)
2135
+ ? creativesRes.creatives.findIndex((c) => c.creativeId === firstUndeliveredItem?.creativeId)
2136
+ : -1;
2137
+ const effectiveDeliveredPrefixLen =
2138
+ firstUndeliveredExtractedIdx >= 0 ? firstUndeliveredExtractedIdx : newlyDeliveredItems.length;
2139
+ effectiveNextPageToken = computeRewoundCursorForDeliveredPrefix(
2140
+ creativesRes.cursorResumeContext,
2141
+ effectiveDeliveredPrefixLen,
2142
+ creativesRes.nextPageToken
2143
+ );
2144
+ effectivePaginationStatus =
2145
+ finalStatus === 'CHARGE_LIMIT_REACHED'
2146
+ ? 'CHARGE_LIMIT_REACHED_HAS_MORE'
2147
+ : finalStatus === 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
2148
+ ? 'DEADLINE_EXCEEDED_HAS_MORE'
2149
+ : 'PARTIAL_BILLING_HALT';
2150
+ effectiveIncompleteReason = finalStatus;
2151
+ } else if (finalStatus === 'CHARGE_LIMIT_REACHED' && effectiveNextPageToken) {
2152
+ effectivePaginationStatus = 'CHARGE_LIMIT_REACHED_HAS_MORE';
2153
+ effectiveIncompleteReason = finalStatus;
2154
+ }
2155
+
2156
+ const paginationMeta = {
2157
+ nextPageToken: effectiveNextPageToken,
2158
+ paginationStatus: effectivePaginationStatus,
2159
+ paginationIncompleteReason: effectiveIncompleteReason,
2160
+ pagesFetched: creativesRes.pagesFetched,
2161
+ ...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
2162
+ ...(undeliveredCount > 0 ? { undeliveredCount } : {}),
2163
+ };
2164
+
2165
+ // Merge reconciled items + newly delivered items with strict creativeId deduplication
2166
+ const allOutputItems = [];
2167
+ const outputSeenIds = new Set();
2168
+ for (const item of [...reconciledItems, ...newlyDeliveredItems]) {
2169
+ if (item?.creativeId && !outputSeenIds.has(item.creativeId)) {
2170
+ outputSeenIds.add(item.creativeId);
2171
+ allOutputItems.push(item);
2172
+ }
2173
+ }
2174
+
2175
+ const simulatedBillableCount = hasChargeHook ? chargedEventCount : allOutputItems.length;
2176
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
2177
+ const telemetry = computeRunEconomicsTelemetry({
2178
+ billableCount: simulatedBillableCount,
2179
+ chargedEventCount,
2180
+ durationMs,
2181
+ requestCount: Math.max(1, totalRpcRequests),
2182
+ unitPriceUsdc: lastResolvedUnitPriceUsdc,
2183
+ actorStartFeeUsdc: hooks?.actorStartFeeUsdc,
2184
+ });
2185
+
2186
+ return finalizeActorReturn({
2187
+ ok: finalOk,
2188
+ status: finalStatus,
2189
+ reason: haltReason,
2190
+ billingMode,
2191
+ queryMeta: entityRes.queryMeta,
2192
+ selectedEntities: entityRes.selectedEntities,
2193
+ items: allOutputItems,
2194
+ extractedCount,
2195
+ deliveredCount,
2196
+ billableCount: simulatedBillableCount,
2197
+ chargedEventCount,
2198
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
2199
+ chargeReceipts,
2200
+ pagination: paginationMeta,
2201
+ ...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
2202
+ cursorResumeContext: creativesRes.cursorResumeContext,
2203
+ telemetry,
2204
+ });
2205
+ }
2206
+
2207
+ export const scrapeGoogleAdsTransparency = runGoogleAdsTransparencyActor;
2208
+