@revclamp/google-ads-transparency-mcp-actor 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1714 @@
1
+ /**
2
+ * Apify Pay-per-Event Execution, Native Platform Adapter & Crash-Safe Billing State Machine (src/apify_actor.mjs)
3
+ *
4
+ * Enforces deterministic Pay-per-Event billing and delivery invariants:
5
+ * 1. Explicit Accounting Separation: Tracks `extractedCount`, `deliveredCount`,
6
+ * `billableCount` (simulated unit-economics count), and confirmed `chargedEventCount`
7
+ * (`0` in local/MCP unbilled mode; strictly validated integer in Apify billing mode).
8
+ * 2. Native Zero-Dependency Apify Platform Adapter (`createApifyPlatformHooks` & `loadApifyActorInput`):
9
+ * Wires Apify Key-Value Store (`INPUT` & `ADPULSE_BILLING_CHECKPOINT`), Dataset (`pushData`),
10
+ * and Pay-per-Event Charging (`POST /v2/actor-runs/:runId/charge` with `Idempotency-Key`)
11
+ * when running on Apify (`APIFY_IS_AT_HOME=1` / `APIFY_TOKEN` + `APIFY_ACTOR_RUN_ID`) or local storage.
12
+ * 3. Crash-Safe KVS Checkpoint State Machine (`inFlightIds` -> `chargedIds` -> `deliveredIds`
13
+ * or `unknownOutcomeIds`), deterministic prefix delivery when `eventChargeLimitReached` returns
14
+ * partial `chargedCount`, cursor rewind on mid-stream halt/limit, and honest zero-event telemetry.
15
+ */
16
+
17
+ import fs from 'node:fs';
18
+ import path from 'node:path';
19
+ import crypto from 'node:crypto';
20
+ import {
21
+ resolveAdvertiserEntities,
22
+ fetchAdvertiserCreatives,
23
+ getCreativeDetailsById,
24
+ createEmptyUntrustedAdContent,
25
+ computeRewoundCursorForDeliveredPrefix,
26
+ verifyEgressUrlSsrfSafe,
27
+ CREATIVE_ID_RE,
28
+ ADVERTISER_ID_RE,
29
+ DEFAULT_RUN_DEADLINE_MS,
30
+ } from './rpc_client.mjs';
31
+
32
+ export const ACTOR_PRICING_CONTRACT = Object.freeze({
33
+ eventName: 'ad-creative-scraped',
34
+ actorStartFeeUsdc: 0.001,
35
+ priceUsdc: 0.0012,
36
+ costCeilingUsdc: 0.0001,
37
+ platformFeeRate: 0.2,
38
+ netPriceUsdc: 0.00096,
39
+ minMarginMultiplier: 2.5,
40
+ computeRatePerGbHourUsdc: 0.3,
41
+ maxMemoryMb: 256,
42
+ maxTimeoutSec: 25,
43
+ chunkSize: 25,
44
+ checkpointKey: 'ADPULSE_BILLING_CHECKPOINT',
45
+ });
46
+
47
+ /**
48
+ * Computes content-addressed batch idempotency key (`sha256(runId + '\x1f' + sortedChunkIds)`).
49
+ *
50
+ * @param {string} runId
51
+ * @param {string[]} creativeIds
52
+ * @returns {string}
53
+ */
54
+ export function computeChunkIdempotencyKey(runId, creativeIds) {
55
+ const safeRunId = String(runId || 'default_run');
56
+ const sortedIds = [...creativeIds].map((id) => String(id)).sort();
57
+ return crypto
58
+ .createHash('sha256')
59
+ .update(`${safeRunId}\x1f${sortedIds.join('\x1f')}`, 'utf8')
60
+ .digest('hex');
61
+ }
62
+
63
+ /**
64
+ * Computes unit-economics and margin telemetry for a single Actor/MCP execution run.
65
+ * Uses Apify's standard `$0.30 / GB-hour` compute rate (`0.25 GB` at `256MB`).
66
+ * When `billableCount === 0`, returns `economicsStatus: 'NO_BILLABLE_EVENTS'` with
67
+ * `withinCostCeiling: false` and `marginPositive: false`.
68
+ *
69
+ * @param {object} params
70
+ * @param {number} params.billableCount
71
+ * @param {number} [params.chargedEventCount=0]
72
+ * @param {number} params.durationMs
73
+ * @param {number} [params.memoryMb=256]
74
+ * @param {number} [params.requestCount=1]
75
+ * @returns {object}
76
+ */
77
+ export function computeRunEconomicsTelemetry({
78
+ billableCount = 0,
79
+ chargedEventCount = 0,
80
+ durationMs = 100,
81
+ memoryMb =
82
+ Number(process.env.ACTOR_MEMORY_MBYTES || process.env.APIFY_MEMORY_MBYTES) ||
83
+ ACTOR_PRICING_CONTRACT.maxMemoryMb,
84
+ requestCount = 1,
85
+ } = {}) {
86
+ const safeCount = Math.max(0, Math.floor(Number(billableCount) || 0));
87
+ const safeChargedCount = Math.max(0, Math.floor(Number(chargedEventCount) || 0));
88
+ const safeDurationMs = Math.max(1, Number(durationMs) || 1);
89
+ const safeMemoryMb = Math.min(1024, Math.max(64, Number(memoryMb) || 256));
90
+ const safeRequests = Math.max(1, Math.floor(Number(requestCount) || 1));
91
+
92
+ // Apify compute rate: $0.30 per GB-hour; 256MB = 0.25 GB -> $0.075/hour -> $0.00002083/sec
93
+ const computeHours = safeDurationMs / 3600000;
94
+ const computeCostUsdc =
95
+ (safeMemoryMb / 1024) * computeHours * ACTOR_PRICING_CONTRACT.computeRatePerGbHourUsdc;
96
+ const networkOverheadUsdc = safeRequests * 0.0000002;
97
+ const totalRunCostUsdc = Number((computeCostUsdc + networkOverheadUsdc).toFixed(8));
98
+ const actorStartFeeUsdc = ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
99
+
100
+ const configuredFloorMultiplier = Number(
101
+ (ACTOR_PRICING_CONTRACT.netPriceUsdc / ACTOR_PRICING_CONTRACT.costCeilingUsdc).toFixed(2)
102
+ );
103
+
104
+ if (safeCount === 0) {
105
+ return {
106
+ economicsStatus: 'NO_BILLABLE_EVENTS',
107
+ billableCount: 0,
108
+ chargedEventCount: safeChargedCount,
109
+ durationMs: safeDurationMs,
110
+ memoryMb: safeMemoryMb,
111
+ requestCount: safeRequests,
112
+ actorStartFeeUsdc,
113
+ totalRunCostUsdc,
114
+ actualCostPerEventUsdc: 0,
115
+ costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
116
+ withinCostCeiling: false,
117
+ grossRevenueUsdc: 0,
118
+ netRevenueUsdc: 0,
119
+ configuredFloorMultiplier,
120
+ effectiveMarginMultiplier: 0,
121
+ marginPositive: false,
122
+ };
123
+ }
124
+
125
+ const actualCostPerEventUsdc = Number((totalRunCostUsdc / safeCount).toFixed(8));
126
+ const grossRevenueUsdc = Number((safeCount * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6));
127
+ const netRevenueUsdc = Number((safeCount * ACTOR_PRICING_CONTRACT.netPriceUsdc).toFixed(6));
128
+ const effectiveMarginMultiplier =
129
+ actualCostPerEventUsdc > 0
130
+ ? Number((ACTOR_PRICING_CONTRACT.netPriceUsdc / actualCostPerEventUsdc).toFixed(2))
131
+ : configuredFloorMultiplier;
132
+
133
+ const withinCostCeiling = actualCostPerEventUsdc <= ACTOR_PRICING_CONTRACT.costCeilingUsdc;
134
+ const marginPositive =
135
+ withinCostCeiling &&
136
+ configuredFloorMultiplier >= ACTOR_PRICING_CONTRACT.minMarginMultiplier;
137
+
138
+ return {
139
+ economicsStatus: marginPositive ? 'PROFITABLE' : 'COST_CEILING_EXCEEDED',
140
+ billableCount: safeCount,
141
+ chargedEventCount: safeChargedCount,
142
+ durationMs: safeDurationMs,
143
+ memoryMb: safeMemoryMb,
144
+ requestCount: safeRequests,
145
+ actorStartFeeUsdc,
146
+ totalRunCostUsdc,
147
+ actualCostPerEventUsdc,
148
+ costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
149
+ withinCostCeiling,
150
+ grossRevenueUsdc,
151
+ netRevenueUsdc,
152
+ configuredFloorMultiplier,
153
+ effectiveMarginMultiplier,
154
+ marginPositive,
155
+ };
156
+ }
157
+
158
+ /**
159
+ * Validates that an Apify API base URL is safe (`https://api.apify.com` by default).
160
+ * When `rawApiBaseUrl` is an internal HTTP endpoint on Apify Cloud and a valid HTTPS
161
+ * `fallbackPublicBaseUrl` (`APIFY_API_PUBLIC_BASE_URL`) is provided, uses the HTTPS public URL.
162
+ *
163
+ * @param {string} [rawApiBaseUrl]
164
+ * @param {string} [fallbackPublicBaseUrl]
165
+ * @returns {string}
166
+ */
167
+ function resolveSafeApifyApiBaseUrl(rawApiBaseUrl, fallbackPublicBaseUrl = undefined) {
168
+ const candidate = String(rawApiBaseUrl || fallbackPublicBaseUrl || 'https://api.apify.com')
169
+ .trim()
170
+ .replace(/\/+$/, '');
171
+ const ssrf = verifyEgressUrlSsrfSafe(candidate);
172
+ if (!ssrf.safe) {
173
+ if (fallbackPublicBaseUrl) {
174
+ const publicCandidate = String(fallbackPublicBaseUrl).trim().replace(/\/+$/, '');
175
+ const publicSsrf = verifyEgressUrlSsrfSafe(publicCandidate);
176
+ if (publicSsrf.safe) {
177
+ return publicCandidate;
178
+ }
179
+ }
180
+ throw new Error(`Invalid or unsafe APIFY_API_BASE_URL '${candidate}': ${ssrf.reason}`);
181
+ }
182
+ return candidate;
183
+ }
184
+
185
+ /**
186
+ * Loads Actor input JSON from CLI `--input-json`, `APIFY_INPUT_PATH`, local Apify storage
187
+ * (`$ACTOR_STORAGE_DIR` / `$APIFY_LOCAL_STORAGE_DIR/key_value_stores/default/INPUT.json`),
188
+ * or Apify Platform KVS REST API.
189
+ *
190
+ * @param {string[]} [cliArgs=[]]
191
+ * @param {object} [env=process.env]
192
+ * @param {Function} [fetchImpl=globalThis.fetch]
193
+ * @returns {Promise<object>}
194
+ */
195
+ export async function loadApifyActorInput(
196
+ cliArgs = [],
197
+ env = process.env,
198
+ fetchImpl = globalThis.fetch
199
+ ) {
200
+ const inputIdx = Array.isArray(cliArgs) ? cliArgs.indexOf('--input-json') : -1;
201
+ if (inputIdx !== -1 && cliArgs[inputIdx + 1]) {
202
+ const raw = fs.readFileSync(cliArgs[inputIdx + 1], 'utf8');
203
+ return JSON.parse(raw);
204
+ }
205
+
206
+ if (env?.APIFY_INPUT_PATH && fs.existsSync(env.APIFY_INPUT_PATH)) {
207
+ const raw = fs.readFileSync(env.APIFY_INPUT_PATH, 'utf8');
208
+ return JSON.parse(raw);
209
+ }
210
+
211
+ const localStorageDir =
212
+ env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
213
+ if (localStorageDir) {
214
+ const kvsId =
215
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ||
216
+ env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID ||
217
+ 'default';
218
+ const inputKey = env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT';
219
+ for (const candidateName of [`${inputKey}.json`, inputKey]) {
220
+ const localInputPath = path.join(
221
+ localStorageDir,
222
+ 'key_value_stores',
223
+ kvsId,
224
+ candidateName
225
+ );
226
+ if (fs.existsSync(localInputPath)) {
227
+ return JSON.parse(fs.readFileSync(localInputPath, 'utf8'));
228
+ }
229
+ }
230
+ }
231
+
232
+ const kvsId =
233
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
234
+ const token = env?.APIFY_TOKEN;
235
+ if (kvsId && token && typeof fetchImpl === 'function') {
236
+ const apiBase = resolveSafeApifyApiBaseUrl(
237
+ env?.APIFY_API_BASE_URL,
238
+ env?.APIFY_API_PUBLIC_BASE_URL
239
+ );
240
+ const inputKey = encodeURIComponent(
241
+ env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT'
242
+ );
243
+ const url = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${inputKey}`;
244
+ const controller = new AbortController();
245
+ const timer = setTimeout(() => controller.abort(), 4000);
246
+ try {
247
+ const res = await fetchImpl(url, {
248
+ method: 'GET',
249
+ headers: {
250
+ authorization: `Bearer ${token}`,
251
+ accept: 'application/json',
252
+ },
253
+ signal: controller.signal,
254
+ });
255
+ if (res.status === 200) {
256
+ const text = await res.text();
257
+ if (text && text.trim()) {
258
+ return JSON.parse(text);
259
+ }
260
+ }
261
+ } finally {
262
+ clearTimeout(timer);
263
+ }
264
+ }
265
+
266
+ return {};
267
+ }
268
+
269
+ /**
270
+ * Creates native zero-dependency Apify Platform runtime hooks (`runId`, `isAtHome`,
271
+ * `loadCheckpoint`, `saveCheckpoint`, `listDeliveredIds`, `pushData`, `chargeEvent`, `saveOutput`)
272
+ * when Apify environment variables (`ACTOR_*` primary, `APIFY_*` fallback) or local
273
+ * storage directories are configured. Returns `{}` in pure local CLI/MCP mode.
274
+ *
275
+ * @param {object} [env=process.env]
276
+ * @param {Function} [fetchImpl=globalThis.fetch]
277
+ * @returns {object}
278
+ */
279
+ export function createApifyPlatformHooks(env = process.env, fetchImpl = globalThis.fetch) {
280
+ const rawRunId = env?.ACTOR_RUN_ID || env?.APIFY_ACTOR_RUN_ID;
281
+ const runId = rawRunId ? String(rawRunId).trim() : undefined;
282
+ const token = env?.APIFY_TOKEN ? String(env.APIFY_TOKEN).trim() : undefined;
283
+ const rawDatasetId = env?.ACTOR_DEFAULT_DATASET_ID || env?.APIFY_DEFAULT_DATASET_ID;
284
+ const datasetId = rawDatasetId ? String(rawDatasetId).trim() : undefined;
285
+ const rawKvsId =
286
+ env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
287
+ const kvsId = rawKvsId ? String(rawKvsId).trim() : undefined;
288
+ const localStorageDir =
289
+ env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
290
+ const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
291
+
292
+ const hasCloudApi = Boolean((isAtHome || runId) && token && typeof fetchImpl === 'function');
293
+ if (!hasCloudApi && !localStorageDir) {
294
+ if (isAtHome) {
295
+ return { isAtHome: true, ...(runId ? { runId } : {}) };
296
+ }
297
+ return runId ? { runId } : {};
298
+ }
299
+
300
+ const hooks = {};
301
+ if (runId) {
302
+ hooks.runId = runId;
303
+ }
304
+ if (isAtHome || hasCloudApi) {
305
+ hooks.isAtHome = true;
306
+ }
307
+
308
+ if (hasCloudApi) {
309
+ const apiBase = resolveSafeApifyApiBaseUrl(
310
+ env?.APIFY_API_BASE_URL,
311
+ env?.APIFY_API_PUBLIC_BASE_URL
312
+ );
313
+ const rawMaxChargeStr =
314
+ env?.ACTOR_MAX_TOTAL_CHARGE_USD !== undefined && env?.ACTOR_MAX_TOTAL_CHARGE_USD !== null
315
+ ? String(env.ACTOR_MAX_TOTAL_CHARGE_USD).trim()
316
+ : '';
317
+ const maxTotalChargeUsdRaw = rawMaxChargeStr !== '' ? Number(rawMaxChargeStr) : NaN;
318
+ const hasMaxChargeUsd = Number.isFinite(maxTotalChargeUsdRaw) && maxTotalChargeUsdRaw >= 0;
319
+ const includeStartFeeFlag = String(env?.ACTOR_INCLUDE_START_FEE ?? '1')
320
+ .trim()
321
+ .toLowerCase();
322
+ const includeStartFee = includeStartFeeFlag !== '0' && includeStartFeeFlag !== 'false';
323
+ const explicitStartFeeUsd =
324
+ env?.ACTOR_START_FEE_USD !== undefined &&
325
+ env?.ACTOR_START_FEE_USD !== null &&
326
+ String(env.ACTOR_START_FEE_USD).trim() !== ''
327
+ ? Number(env.ACTOR_START_FEE_USD)
328
+ : ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
329
+ const startFeeMicros =
330
+ includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
331
+ ? Math.round(explicitStartFeeUsd * 1_000_000)
332
+ : 0;
333
+ const unitPriceMicros = Math.round(ACTOR_PRICING_CONTRACT.priceUsdc * 1_000_000);
334
+ const maxBudgetMicros = hasMaxChargeUsd ? Math.round(maxTotalChargeUsdRaw * 1_000_000) : 0;
335
+ const maxPayableEvents = hasMaxChargeUsd
336
+ ? Math.floor(Math.max(0, maxBudgetMicros - startFeeMicros) / unitPriceMicros)
337
+ : Infinity;
338
+ let cumulativeChargedInAdapter = 0;
339
+ let adapterChargeLimitLatched = false;
340
+
341
+ if (kvsId) {
342
+ const checkpointRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${ACTOR_PRICING_CONTRACT.checkpointKey}`;
343
+ const outputRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/OUTPUT`;
344
+
345
+ hooks.loadCheckpoint = async () => {
346
+ const controller = new AbortController();
347
+ const timer = setTimeout(() => controller.abort(), 250);
348
+ try {
349
+ const res = await fetchImpl(checkpointRecordUrl, {
350
+ method: 'GET',
351
+ headers: {
352
+ authorization: `Bearer ${token}`,
353
+ accept: 'application/json',
354
+ },
355
+ signal: controller.signal,
356
+ });
357
+ if (res.status === 404 || res.status === 204) {
358
+ return null;
359
+ }
360
+ if (res.status >= 200 && res.status < 300) {
361
+ const text = await res.text();
362
+ const loaded = text && text.trim() ? JSON.parse(text) : null;
363
+ if (loaded && Array.isArray(loaded.chargedIds)) {
364
+ cumulativeChargedInAdapter = loaded.chargedIds.length;
365
+ }
366
+ return loaded;
367
+ }
368
+ throw new Error(`Apify KVS loadCheckpoint failed with HTTP ${res.status}`);
369
+ } finally {
370
+ clearTimeout(timer);
371
+ }
372
+ };
373
+
374
+ hooks.saveCheckpoint = async (snapshot) => {
375
+ const controller = new AbortController();
376
+ const timer = setTimeout(() => controller.abort(), 250);
377
+ try {
378
+ const res = await fetchImpl(checkpointRecordUrl, {
379
+ method: 'PUT',
380
+ headers: {
381
+ authorization: `Bearer ${token}`,
382
+ 'content-type': 'application/json; charset=utf-8',
383
+ },
384
+ body: JSON.stringify(snapshot),
385
+ signal: controller.signal,
386
+ });
387
+ if (res.status < 200 || res.status >= 300) {
388
+ throw new Error(`Apify KVS saveCheckpoint HTTP ${res.status}`);
389
+ }
390
+ } finally {
391
+ clearTimeout(timer);
392
+ }
393
+ };
394
+
395
+ hooks.saveOutput = async (outputRecord) => {
396
+ const controller = new AbortController();
397
+ const timer = setTimeout(() => controller.abort(), 250);
398
+ try {
399
+ const res = await fetchImpl(outputRecordUrl, {
400
+ method: 'PUT',
401
+ headers: {
402
+ authorization: `Bearer ${token}`,
403
+ 'content-type': 'application/json; charset=utf-8',
404
+ },
405
+ body: JSON.stringify(outputRecord),
406
+ signal: controller.signal,
407
+ });
408
+ if (res.status < 200 || res.status >= 300) {
409
+ throw new Error(`Apify KVS saveOutput HTTP ${res.status}`);
410
+ }
411
+ } finally {
412
+ clearTimeout(timer);
413
+ }
414
+ };
415
+ }
416
+
417
+ if (datasetId) {
418
+ const datasetItemsUrl = `${apiBase}/v2/datasets/${encodeURIComponent(datasetId)}/items`;
419
+ hooks.listDeliveredIds = async () => {
420
+ const controller = new AbortController();
421
+ const timer = setTimeout(() => controller.abort(), 500);
422
+ try {
423
+ const res = await fetchImpl(`${datasetItemsUrl}?fields=creativeId&clean=1`, {
424
+ method: 'GET',
425
+ headers: {
426
+ authorization: `Bearer ${token}`,
427
+ accept: 'application/json',
428
+ },
429
+ signal: controller.signal,
430
+ });
431
+ if (res.status >= 200 && res.status < 300) {
432
+ const text = await res.text();
433
+ const parsed = text && text.trim() ? JSON.parse(text) : [];
434
+ const arr = Array.isArray(parsed)
435
+ ? parsed
436
+ : Array.isArray(parsed?.data?.items)
437
+ ? parsed.data.items
438
+ : [];
439
+ return arr
440
+ .map((row) => (typeof row?.creativeId === 'string' ? row.creativeId : null))
441
+ .filter(Boolean);
442
+ }
443
+ throw new Error(`Apify Dataset listDeliveredIds failed with HTTP ${res.status}`);
444
+ } finally {
445
+ clearTimeout(timer);
446
+ }
447
+ };
448
+
449
+ hooks.pushData = async (items) => {
450
+ const controller = new AbortController();
451
+ const timer = setTimeout(() => controller.abort(), 500);
452
+ try {
453
+ const res = await fetchImpl(datasetItemsUrl, {
454
+ method: 'POST',
455
+ headers: {
456
+ authorization: `Bearer ${token}`,
457
+ 'content-type': 'application/json; charset=utf-8',
458
+ },
459
+ body: JSON.stringify(items),
460
+ signal: controller.signal,
461
+ });
462
+ if (res.status < 200 || res.status >= 300) {
463
+ throw new Error(`Apify Dataset pushData HTTP ${res.status}`);
464
+ }
465
+ } finally {
466
+ clearTimeout(timer);
467
+ }
468
+ };
469
+ }
470
+
471
+ if (runId) {
472
+ const chargeUrl = `${apiBase}/v2/actor-runs/${encodeURIComponent(runId)}/charge`;
473
+ hooks.chargeEvent = async (receipt) => {
474
+ const requestedCount = Math.max(0, Number(receipt?.count) || 0);
475
+ if (adapterChargeLimitLatched) {
476
+ return {
477
+ chargedCount: 0,
478
+ eventChargeLimitReached: true,
479
+ chargedCreativeIds: [],
480
+ };
481
+ }
482
+ const remainingAllowance = Number.isFinite(maxPayableEvents)
483
+ ? Math.max(0, maxPayableEvents - cumulativeChargedInAdapter)
484
+ : requestedCount;
485
+ const countToCharge = Math.min(requestedCount, remainingAllowance);
486
+ const limitReached =
487
+ Number.isFinite(maxPayableEvents) &&
488
+ cumulativeChargedInAdapter + countToCharge >= maxPayableEvents &&
489
+ countToCharge < requestedCount;
490
+
491
+ if (countToCharge === 0) {
492
+ return {
493
+ chargedCount: 0,
494
+ eventChargeLimitReached: Number.isFinite(maxPayableEvents),
495
+ chargedCreativeIds: [],
496
+ };
497
+ }
498
+
499
+ const controller = new AbortController();
500
+ const timer = setTimeout(() => controller.abort(), 600);
501
+ try {
502
+ const res = await fetchImpl(chargeUrl, {
503
+ method: 'POST',
504
+ headers: {
505
+ authorization: `Bearer ${token}`,
506
+ 'content-type': 'application/json; charset=utf-8',
507
+ 'idempotency-key': receipt.idempotencyKey,
508
+ },
509
+ body: JSON.stringify({
510
+ eventName: receipt.eventName,
511
+ count: countToCharge,
512
+ }),
513
+ signal: controller.signal,
514
+ });
515
+ if (res.status === 402) {
516
+ // Platform enforced spending ceiling (HTTP 402 Payment Required): latch so no further charge POSTs occur
517
+ adapterChargeLimitLatched = true;
518
+ return {
519
+ chargedCount: 0,
520
+ eventChargeLimitReached: true,
521
+ chargedCreativeIds: [],
522
+ };
523
+ }
524
+ if (res.status < 200 || res.status >= 300) {
525
+ throw new Error(`Apify Pay-per-Event charge HTTP ${res.status}`);
526
+ }
527
+ const rawText = await res.text();
528
+ const parsed = rawText && rawText.trim() ? JSON.parse(rawText) : {};
529
+ const dataObj = parsed?.data && typeof parsed.data === 'object' ? parsed.data : parsed;
530
+ // Apify REST `POST /v2/actor-runs/{runId}/charge` returns HTTP 201 Created with `{}` on success.
531
+ // Only fallback to `countToCharge` when `chargedCount` is genuinely omitted; if present but invalid, throw fail-closed.
532
+ const hasExplicitChargedCount =
533
+ Boolean(dataObj) &&
534
+ typeof dataObj === 'object' &&
535
+ Object.prototype.hasOwnProperty.call(dataObj, 'chargedCount') &&
536
+ dataObj.chargedCount !== undefined;
537
+ if (
538
+ hasExplicitChargedCount &&
539
+ (!Number.isInteger(dataObj.chargedCount) ||
540
+ dataObj.chargedCount < 0 ||
541
+ dataObj.chargedCount > countToCharge)
542
+ ) {
543
+ throw new Error(
544
+ `Apify Pay-per-Event charge response contained invalid chargedCount: ${String(dataObj.chargedCount)}`
545
+ );
546
+ }
547
+ const resolvedChargedCount = hasExplicitChargedCount
548
+ ? dataObj.chargedCount
549
+ : countToCharge;
550
+ cumulativeChargedInAdapter += resolvedChargedCount;
551
+ const resolvedLimitReached =
552
+ Boolean(dataObj?.eventChargeLimitReached) ||
553
+ limitReached ||
554
+ (Number.isFinite(maxPayableEvents) && cumulativeChargedInAdapter >= maxPayableEvents);
555
+ const subsetIds = Array.isArray(dataObj?.chargedCreativeIds)
556
+ ? dataObj.chargedCreativeIds
557
+ : Array.isArray(receipt?.creativeIds)
558
+ ? receipt.creativeIds.slice(0, resolvedChargedCount)
559
+ : undefined;
560
+ return {
561
+ chargedCount: resolvedChargedCount,
562
+ eventChargeLimitReached: resolvedLimitReached,
563
+ ...(subsetIds ? { chargedCreativeIds: subsetIds } : {}),
564
+ };
565
+ } finally {
566
+ clearTimeout(timer);
567
+ }
568
+ };
569
+ }
570
+
571
+ return hooks;
572
+ }
573
+
574
+ // Local Apify storage directory mode (`APIFY_LOCAL_STORAGE_DIR`)
575
+ const storeName = kvsId || 'default';
576
+ const dsName = datasetId || 'default';
577
+ const kvsDir = path.join(localStorageDir, 'key_value_stores', storeName);
578
+ const dsDir = path.join(localStorageDir, 'datasets', dsName);
579
+ const checkpointPath = path.join(kvsDir, `${ACTOR_PRICING_CONTRACT.checkpointKey}.json`);
580
+ const outputPath = path.join(kvsDir, 'OUTPUT.json');
581
+
582
+ hooks.loadCheckpoint = async () => {
583
+ if (!fs.existsSync(checkpointPath)) return null;
584
+ return JSON.parse(fs.readFileSync(checkpointPath, 'utf8'));
585
+ };
586
+
587
+ hooks.saveCheckpoint = async (snapshot) => {
588
+ fs.mkdirSync(kvsDir, { recursive: true });
589
+ fs.writeFileSync(checkpointPath, JSON.stringify(snapshot, null, 2), 'utf8');
590
+ };
591
+
592
+ hooks.saveOutput = async (outputRecord) => {
593
+ fs.mkdirSync(kvsDir, { recursive: true });
594
+ fs.writeFileSync(outputPath, JSON.stringify(outputRecord, null, 2), 'utf8');
595
+ };
596
+
597
+ hooks.listDeliveredIds = async () => {
598
+ if (!fs.existsSync(dsDir)) return [];
599
+ const existingFiles = fs.readdirSync(dsDir).filter((f) => f.endsWith('.json'));
600
+ const ids = [];
601
+ for (const file of existingFiles) {
602
+ try {
603
+ const parsed = JSON.parse(fs.readFileSync(path.join(dsDir, file), 'utf8'));
604
+ if (typeof parsed?.creativeId === 'string') {
605
+ ids.push(parsed.creativeId);
606
+ }
607
+ } catch {
608
+ // Ignore unreadable partial files
609
+ }
610
+ }
611
+ return ids;
612
+ };
613
+
614
+ hooks.pushData = async (items) => {
615
+ fs.mkdirSync(dsDir, { recursive: true });
616
+ const existingSeqs = fs
617
+ .readdirSync(dsDir)
618
+ .map((f) => {
619
+ const m = /^(\d+)\.json$/.exec(f);
620
+ return m ? Number(m[1]) : 0;
621
+ })
622
+ .filter((n) => Number.isInteger(n) && n > 0);
623
+ let nextSeq = existingSeqs.length > 0 ? Math.max(...existingSeqs) + 1 : 1;
624
+ const list = Array.isArray(items) ? items : [items];
625
+ for (const item of list) {
626
+ const fileName = `${String(nextSeq++).padStart(9, '0')}.json`;
627
+ fs.writeFileSync(path.join(dsDir, fileName), JSON.stringify(item, null, 2), 'utf8');
628
+ }
629
+ };
630
+
631
+ return hooks;
632
+ }
633
+
634
+ /**
635
+ * Normalizes a persisted KVS billing checkpoint object.
636
+ * If the checkpoint belongs to a different `runId`, discards stale state and starts fresh.
637
+ *
638
+ * @param {unknown} rawState
639
+ * @param {string} runId
640
+ * @returns {object}
641
+ */
642
+ function normalizeBillingCheckpointState(rawState, runId) {
643
+ const s = rawState && typeof rawState === 'object' && !Array.isArray(rawState) ? rawState : {};
644
+ if (s.runId && String(s.runId).trim() !== String(runId).trim()) {
645
+ return {
646
+ runId: String(runId || 'default_run'),
647
+ chargedIds: [],
648
+ deliveredIds: [],
649
+ inFlightIds: [],
650
+ unknownOutcomeIds: [],
651
+ stagedItemsById: {},
652
+ };
653
+ }
654
+
655
+ const toUniqueIdArray = (arr) =>
656
+ Array.isArray(arr)
657
+ ? Array.from(new Set(arr.map((x) => String(x || '').trim()).filter((x) => CREATIVE_ID_RE.test(x))))
658
+ : [];
659
+
660
+ const stagedItemsById = {};
661
+ if (s.stagedItemsById && typeof s.stagedItemsById === 'object' && !Array.isArray(s.stagedItemsById)) {
662
+ for (const [k, v] of Object.entries(s.stagedItemsById)) {
663
+ if (CREATIVE_ID_RE.test(k) && v && typeof v === 'object') {
664
+ stagedItemsById[k] = v;
665
+ }
666
+ }
667
+ }
668
+
669
+ return {
670
+ runId: String(s.runId || runId || 'default_run'),
671
+ chargedIds: toUniqueIdArray(s.chargedIds),
672
+ deliveredIds: toUniqueIdArray(s.deliveredIds),
673
+ inFlightIds: toUniqueIdArray(s.inFlightIds),
674
+ unknownOutcomeIds: toUniqueIdArray(s.unknownOutcomeIds),
675
+ stagedItemsById,
676
+ };
677
+ }
678
+
679
+ /**
680
+ * Executes the end-to-end Google Ads Transparency Actor workflow with crash-safe billing
681
+ * checkpointing, strict `ChargeResult` validation, deterministic partial-limit prefix delivery,
682
+ * mid-stream cursor rewind, and fail-closed zero-charge protection.
683
+ *
684
+ * @param {object} input
685
+ * @param {object} [hooks]
686
+ * @param {Function} [hooks.pushData]
687
+ * @param {Function} [hooks.chargeEvent]
688
+ * @param {Function} [hooks.loadCheckpoint]
689
+ * @param {Function} [hooks.saveCheckpoint]
690
+ * @param {Function} [hooks.fetchImpl]
691
+ * @param {Function} [hooks.nowMs]
692
+ * @param {string} [hooks.runId]
693
+ * @returns {Promise<object>}
694
+ */
695
+ export async function runGoogleAdsTransparencyActor(input = {}, hooks = {}) {
696
+ const nowMs = typeof hooks.nowMs === 'function' ? hooks.nowMs : () => Date.now();
697
+ const startedAtMs = nowMs();
698
+ // Authoritative runId precedence: hooks.runId -> ACTOR_RUN_ID -> APIFY_ACTOR_RUN_ID -> input.runId -> 'default_run'
699
+ const runId = String(
700
+ hooks.runId ||
701
+ process.env.ACTOR_RUN_ID ||
702
+ process.env.APIFY_ACTOR_RUN_ID ||
703
+ input?.runId ||
704
+ 'default_run'
705
+ );
706
+ const hasChargeHook = typeof hooks.chargeEvent === 'function';
707
+ const billingMode =
708
+ hasChargeHook || hooks.isAtHome === true ? 'APIFY_PAY_PER_EVENT' : 'LOCAL_UNBILLED';
709
+
710
+ const maxRunDurationMs = Math.min(
711
+ DEFAULT_RUN_DEADLINE_MS,
712
+ Math.max(1000, Number(input?.maxRunDurationMs) || DEFAULT_RUN_DEADLINE_MS)
713
+ );
714
+ const deadlineAtMs = startedAtMs + maxRunDurationMs;
715
+ const billingReserveMs =
716
+ maxRunDurationMs >= 6000 ? 2500 : Math.min(1000, Math.floor(maxRunDurationMs * 0.25));
717
+ const rpcDeadlineAtMs = hasChargeHook
718
+ ? Math.max(startedAtMs + 500, deadlineAtMs - billingReserveMs)
719
+ : deadlineAtMs;
720
+ const step4ReserveMs = hasChargeHook
721
+ ? Math.min(2100, Math.floor(billingReserveMs * 0.85))
722
+ : 0;
723
+
724
+ const finalizeActorReturn = async (finalResult) => {
725
+ if (typeof hooks.saveOutput === 'function' && nowMs() < deadlineAtMs) {
726
+ try {
727
+ await hooks.saveOutput({
728
+ ok: finalResult.ok,
729
+ status: finalResult.status,
730
+ reason: finalResult.reason || null,
731
+ billingMode: finalResult.billingMode,
732
+ extractedCount: finalResult.extractedCount,
733
+ deliveredCount: finalResult.deliveredCount,
734
+ billableCount: finalResult.billableCount,
735
+ chargedEventCount: finalResult.chargedEventCount,
736
+ pagination: finalResult.pagination,
737
+ ...(Array.isArray(finalResult.warnings) && finalResult.warnings.length > 0
738
+ ? { warnings: finalResult.warnings }
739
+ : {}),
740
+ queryMeta: finalResult.queryMeta,
741
+ selectedEntities: finalResult.selectedEntities,
742
+ telemetry: finalResult.telemetry,
743
+ });
744
+ } catch {
745
+ // Non-fatal KVS summary write
746
+ }
747
+ }
748
+ return finalResult;
749
+ };
750
+
751
+ const pushDataWithRetry = async (itemsToPush) => {
752
+ if (
753
+ typeof hooks.pushData !== 'function' ||
754
+ !Array.isArray(itemsToPush) ||
755
+ itemsToPush.length === 0
756
+ ) {
757
+ return { ok: true };
758
+ }
759
+ let pendingItems = [...itemsToPush];
760
+ let lastErr = null;
761
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
762
+ try {
763
+ await hooks.pushData(pendingItems);
764
+ return { ok: true };
765
+ } catch (err) {
766
+ lastErr = err;
767
+ if (attempt < 2) {
768
+ if (nowMs() + 1500 >= deadlineAtMs || typeof hooks.listDeliveredIds !== 'function') {
769
+ return { ok: false, error: lastErr };
770
+ }
771
+ try {
772
+ const committedIds = new Set(await hooks.listDeliveredIds());
773
+ pendingItems = pendingItems.filter((it) => !committedIds.has(it.creativeId));
774
+ if (pendingItems.length === 0) {
775
+ return { ok: true };
776
+ }
777
+ } catch (reconcileErr) {
778
+ return { ok: false, error: reconcileErr || lastErr };
779
+ }
780
+ }
781
+ }
782
+ }
783
+ return { ok: false, error: lastErr };
784
+ };
785
+
786
+ // Step 0.0: Fail-Closed At-Home Cloud Hook Completeness Gate
787
+ // In Apify Cloud (`hooks.isAtHome === true`), require all 4 platform hooks (`chargeEvent`,
788
+ // `pushData`, `loadCheckpoint`, `saveCheckpoint`) before any RPC fetch or billing occurs.
789
+ if (
790
+ hooks.isAtHome === true &&
791
+ (typeof hooks.chargeEvent !== 'function' ||
792
+ typeof hooks.pushData !== 'function' ||
793
+ typeof hooks.loadCheckpoint !== 'function' ||
794
+ typeof hooks.saveCheckpoint !== 'function')
795
+ ) {
796
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
797
+ return finalizeActorReturn({
798
+ ok: false,
799
+ status: 'PARTIAL_BILLING_HALT',
800
+ reason:
801
+ 'Incomplete Apify Cloud runtime hooks (missing chargeEvent, pushData, loadCheckpoint, or saveCheckpoint in at-home mode)',
802
+ billingMode,
803
+ queryMeta: null,
804
+ selectedEntities: [],
805
+ items: [],
806
+ extractedCount: 0,
807
+ deliveredCount: 0,
808
+ billableCount: 0,
809
+ chargedEventCount: 0,
810
+ unknownOutcomeIds: [],
811
+ chargeReceipts: [],
812
+ pagination: {
813
+ nextPageToken: input?.pageToken || null,
814
+ paginationStatus: 'PARTIAL_BILLING_HALT',
815
+ paginationIncompleteReason: 'CLOUD_HOOKS_INCOMPLETE',
816
+ },
817
+ telemetry: computeRunEconomicsTelemetry({
818
+ billableCount: 0,
819
+ chargedEventCount: 0,
820
+ durationMs,
821
+ requestCount: 1,
822
+ }),
823
+ });
824
+ }
825
+
826
+ // Step 0: Load & Reconcile Crash-Safe Billing Checkpoint (fail-closed if loadCheckpoint throws)
827
+ let rawLoadedCheckpoint = null;
828
+ if (typeof hooks.loadCheckpoint === 'function') {
829
+ try {
830
+ rawLoadedCheckpoint = await hooks.loadCheckpoint();
831
+ } catch (cpErr) {
832
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
833
+ return finalizeActorReturn({
834
+ ok: false,
835
+ status: 'PARTIAL_BILLING_HALT',
836
+ reason: `Failed to load billing checkpoint fail-closed: ${cpErr?.message || String(cpErr)}`,
837
+ billingMode,
838
+ queryMeta: null,
839
+ selectedEntities: [],
840
+ items: [],
841
+ extractedCount: 0,
842
+ deliveredCount: 0,
843
+ billableCount: 0,
844
+ chargedEventCount: 0,
845
+ unknownOutcomeIds: [],
846
+ chargeReceipts: [],
847
+ pagination: {
848
+ nextPageToken: input?.pageToken || null,
849
+ paginationStatus: 'PARTIAL_BILLING_HALT',
850
+ paginationIncompleteReason: 'CHECKPOINT_LOAD_FAILED',
851
+ },
852
+ telemetry: computeRunEconomicsTelemetry({
853
+ billableCount: 0,
854
+ chargedEventCount: 0,
855
+ durationMs,
856
+ requestCount: 1,
857
+ }),
858
+ });
859
+ }
860
+ }
861
+ const checkpoint = normalizeBillingCheckpointState(rawLoadedCheckpoint, runId);
862
+
863
+ const chargedSet = new Set(checkpoint.chargedIds);
864
+ const deliveredSet = new Set(checkpoint.deliveredIds);
865
+ const unknownOutcomeSet = new Set(checkpoint.unknownOutcomeIds);
866
+ let inFlightSet = new Set(checkpoint.inFlightIds);
867
+ const stagedItemsById = { ...checkpoint.stagedItemsById };
868
+
869
+ const persistCheckpoint = async () => {
870
+ const snapshot = {
871
+ runId,
872
+ chargedIds: Array.from(chargedSet),
873
+ deliveredIds: Array.from(deliveredSet),
874
+ inFlightIds: Array.from(inFlightSet),
875
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
876
+ stagedItemsById,
877
+ };
878
+ if (typeof hooks.saveCheckpoint === 'function') {
879
+ await hooks.saveCheckpoint(snapshot);
880
+ }
881
+ return snapshot;
882
+ };
883
+
884
+ const reconciledItems = [];
885
+ let deliveredCount = deliveredSet.size;
886
+
887
+ // Step 0a: Reconcile any already-paid items (`chargedIds \ deliveredIds`) without re-charging.
888
+ // First check `hooks.listDeliveredIds()` if available so items already committed to dataset before a lost ACK
889
+ // are marked delivered without duplicate dataset POSTs. Fail closed if dataset listing throws.
890
+ const paidUndeliveredIds = Array.from(chargedSet).filter((id) => !deliveredSet.has(id));
891
+ if (paidUndeliveredIds.length > 0) {
892
+ let alreadyInDatasetSet = new Set();
893
+ if (typeof hooks.listDeliveredIds === 'function') {
894
+ try {
895
+ const existingDsIds = await hooks.listDeliveredIds();
896
+ if (Array.isArray(existingDsIds)) {
897
+ alreadyInDatasetSet = new Set(existingDsIds);
898
+ }
899
+ } catch (dsListErr) {
900
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
901
+ return finalizeActorReturn({
902
+ ok: false,
903
+ status: 'PARTIAL_BILLING_HALT',
904
+ reason: `Failed to verify delivered dataset IDs during reconciliation: ${dsListErr?.message || String(dsListErr)}`,
905
+ billingMode,
906
+ queryMeta: null,
907
+ selectedEntities: [],
908
+ items: [],
909
+ extractedCount: 0,
910
+ deliveredCount,
911
+ billableCount: 0,
912
+ chargedEventCount: chargedSet.size,
913
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
914
+ chargeReceipts: [],
915
+ pagination: {
916
+ nextPageToken: input?.pageToken || null,
917
+ paginationStatus: 'PARTIAL_BILLING_HALT',
918
+ paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
919
+ },
920
+ telemetry: computeRunEconomicsTelemetry({
921
+ billableCount: 0,
922
+ chargedEventCount: chargedSet.size,
923
+ durationMs,
924
+ requestCount: 1,
925
+ }),
926
+ });
927
+ }
928
+ }
929
+
930
+ const deliverablePairs = [];
931
+ for (const id of paidUndeliveredIds) {
932
+ const staged = stagedItemsById[id];
933
+ if (alreadyInDatasetSet.has(id)) {
934
+ deliveredSet.add(id);
935
+ if (staged && typeof staged === 'object' && staged.creativeId === id) {
936
+ reconciledItems.push(staged);
937
+ }
938
+ } else if (staged && typeof staged === 'object' && staged.creativeId === id) {
939
+ deliverablePairs.push({ id, item: staged });
940
+ } else {
941
+ unknownOutcomeSet.add(id);
942
+ }
943
+ }
944
+
945
+ if (deliverablePairs.length > 0) {
946
+ const itemsToDeliver = deliverablePairs.map((p) => p.item);
947
+ const pushRes = await pushDataWithRetry(itemsToDeliver);
948
+ if (!pushRes.ok) {
949
+ await persistCheckpoint();
950
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
951
+ return finalizeActorReturn({
952
+ ok: false,
953
+ status: 'PARTIAL_BILLING_HALT',
954
+ reason: `Dataset pushData failed during reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
955
+ billingMode,
956
+ queryMeta: null,
957
+ selectedEntities: [],
958
+ items: reconciledItems,
959
+ extractedCount: 0,
960
+ deliveredCount,
961
+ billableCount: reconciledItems.length,
962
+ chargedEventCount: chargedSet.size,
963
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
964
+ chargeReceipts: [],
965
+ pagination: {
966
+ nextPageToken: input?.pageToken || null,
967
+ paginationStatus: 'PARTIAL_BILLING_HALT',
968
+ paginationIncompleteReason: 'DATASET_PUSH_FAILED',
969
+ },
970
+ telemetry: computeRunEconomicsTelemetry({
971
+ billableCount: reconciledItems.length,
972
+ chargedEventCount: chargedSet.size,
973
+ durationMs,
974
+ requestCount: 1,
975
+ }),
976
+ });
977
+ }
978
+ for (const { id, item } of deliverablePairs) {
979
+ deliveredSet.add(id);
980
+ reconciledItems.push(item);
981
+ }
982
+ }
983
+ deliveredCount = deliveredSet.size;
984
+ await persistCheckpoint();
985
+ }
986
+
987
+ // Step 0b: Any leftover `inFlightIds` from a prior interrupted run transition to `unknownOutcomeIds`
988
+ if (inFlightSet.size > 0) {
989
+ for (const id of inFlightSet) {
990
+ unknownOutcomeSet.add(id);
991
+ }
992
+ inFlightSet = new Set();
993
+ await persistCheckpoint();
994
+ }
995
+
996
+ if (unknownOutcomeSet.size > 0 && input?.strictBillingResume !== false) {
997
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
998
+ return finalizeActorReturn({
999
+ ok: false,
1000
+ status: 'PARTIAL_BILLING_HALT',
1001
+ reason: `Halted due to ${unknownOutcomeSet.size} unreconciled creative(s) in UNKNOWN_CHARGE_OUTCOME state`,
1002
+ billingMode,
1003
+ queryMeta: null,
1004
+ selectedEntities: [],
1005
+ items: reconciledItems,
1006
+ extractedCount: 0,
1007
+ deliveredCount,
1008
+ billableCount: reconciledItems.length,
1009
+ chargedEventCount: chargedSet.size,
1010
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1011
+ chargeReceipts: [],
1012
+ pagination: {
1013
+ nextPageToken: input?.pageToken || null,
1014
+ paginationStatus: 'PARTIAL_BILLING_HALT',
1015
+ paginationIncompleteReason: 'UNKNOWN_CHARGE_OUTCOME',
1016
+ },
1017
+ telemetry: computeRunEconomicsTelemetry({
1018
+ billableCount: reconciledItems.length,
1019
+ chargedEventCount: chargedSet.size,
1020
+ durationMs,
1021
+ requestCount: 1,
1022
+ }),
1023
+ });
1024
+ }
1025
+
1026
+ const sleepFn =
1027
+ typeof hooks.sleepFn === 'function'
1028
+ ? hooks.sleepFn
1029
+ : typeof input?.sleepFn === 'function'
1030
+ ? input.sleepFn
1031
+ : undefined;
1032
+ const randomFn =
1033
+ typeof hooks.randomFn === 'function'
1034
+ ? hooks.randomFn
1035
+ : typeof input?.randomFn === 'function'
1036
+ ? input.randomFn
1037
+ : undefined;
1038
+
1039
+ // Step 1: Resolve Advertiser Entities
1040
+ const entityRes = await resolveAdvertiserEntities({
1041
+ query: input?.query,
1042
+ advertiserId: input?.advertiserId,
1043
+ advertiserName: input?.advertiserName,
1044
+ countryCode: input?.countryCode,
1045
+ includeSubsidiaries: input?.includeSubsidiaries,
1046
+ maxEntities: input?.maxEntities,
1047
+ fetchImpl: hooks.fetchImpl,
1048
+ nowFn: nowMs,
1049
+ sleepFn,
1050
+ randomFn,
1051
+ timeoutMs: input?.timeoutMs,
1052
+ maxRetries: input?.maxRetries,
1053
+ deadlineAtMs: rpcDeadlineAtMs,
1054
+ });
1055
+
1056
+ if (!entityRes.ok) {
1057
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1058
+ const isPreChargeDeadline =
1059
+ hasChargeHook &&
1060
+ nowMs() >= rpcDeadlineAtMs &&
1061
+ /deadline exceeded/i.test(String(entityRes.reason || ''));
1062
+ return finalizeActorReturn({
1063
+ ok: false,
1064
+ status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : entityRes.status,
1065
+ reason: entityRes.reason,
1066
+ billingMode,
1067
+ queryMeta: entityRes.queryMeta,
1068
+ selectedEntities: [],
1069
+ items: reconciledItems,
1070
+ extractedCount: 0,
1071
+ deliveredCount,
1072
+ billableCount: reconciledItems.length,
1073
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1074
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1075
+ chargeReceipts: [],
1076
+ pagination: {
1077
+ nextPageToken: input?.pageToken || null,
1078
+ paginationStatus: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_HAS_MORE' : entityRes.status,
1079
+ paginationIncompleteReason: isPreChargeDeadline
1080
+ ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
1081
+ : entityRes.reason || null,
1082
+ },
1083
+ telemetry: computeRunEconomicsTelemetry({
1084
+ billableCount: reconciledItems.length,
1085
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1086
+ durationMs,
1087
+ requestCount: Math.max(1, entityRes.rpcAttempts),
1088
+ }),
1089
+ });
1090
+ }
1091
+
1092
+ if (entityRes.status === 'EMPTY_RESULT' || entityRes.selectedEntities.length === 0) {
1093
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1094
+ return finalizeActorReturn({
1095
+ ok: true,
1096
+ status: 'EMPTY_RESULT',
1097
+ billingMode,
1098
+ queryMeta: entityRes.queryMeta,
1099
+ selectedEntities: [],
1100
+ items: reconciledItems,
1101
+ extractedCount: 0,
1102
+ deliveredCount,
1103
+ billableCount: reconciledItems.length,
1104
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1105
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1106
+ chargeReceipts: [],
1107
+ pagination: {
1108
+ nextPageToken: null,
1109
+ paginationStatus: 'EXHAUSTED',
1110
+ paginationIncompleteReason: null,
1111
+ },
1112
+ telemetry: computeRunEconomicsTelemetry({
1113
+ billableCount: reconciledItems.length,
1114
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1115
+ durationMs,
1116
+ requestCount: Math.max(1, entityRes.rpcAttempts),
1117
+ }),
1118
+ });
1119
+ }
1120
+
1121
+ // Step 2: Fetch Creatives across selected verified advertiser entities
1122
+ const advertiserIds = entityRes.selectedEntities.map((e) => e.advertiserId);
1123
+ const creativesRes = await fetchAdvertiserCreatives({
1124
+ advertiserIds,
1125
+ entities: entityRes.selectedEntities,
1126
+ maxResults: input?.maxResults,
1127
+ maxPages: input?.maxPages,
1128
+ pageSize: input?.pageSize,
1129
+ pageToken: input?.pageToken,
1130
+ countryCode: input?.countryCode,
1131
+ formatFilter: input?.formatFilter,
1132
+ fetchImpl: hooks.fetchImpl,
1133
+ nowFn: nowMs,
1134
+ sleepFn,
1135
+ randomFn,
1136
+ timeoutMs: input?.timeoutMs,
1137
+ maxRetries: input?.maxRetries,
1138
+ deadlineAtMs: rpcDeadlineAtMs,
1139
+ });
1140
+
1141
+ let totalRpcRequests = entityRes.rpcAttempts + creativesRes.rpcAttempts;
1142
+
1143
+ if (!creativesRes.ok) {
1144
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1145
+ const isPreChargeDeadline =
1146
+ hasChargeHook &&
1147
+ nowMs() >= rpcDeadlineAtMs &&
1148
+ /deadline exceeded/i.test(String(creativesRes.reason || ''));
1149
+ return finalizeActorReturn({
1150
+ ok: false,
1151
+ status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : creativesRes.status,
1152
+ reason: creativesRes.reason,
1153
+ billingMode,
1154
+ queryMeta: entityRes.queryMeta,
1155
+ selectedEntities: entityRes.selectedEntities,
1156
+ items: reconciledItems,
1157
+ extractedCount: 0,
1158
+ deliveredCount,
1159
+ billableCount: reconciledItems.length,
1160
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1161
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1162
+ chargeReceipts: [],
1163
+ pagination: {
1164
+ nextPageToken: input?.pageToken || null,
1165
+ paginationStatus: isPreChargeDeadline
1166
+ ? 'DEADLINE_EXCEEDED_HAS_MORE'
1167
+ : creativesRes.paginationStatus || creativesRes.status,
1168
+ paginationIncompleteReason: isPreChargeDeadline
1169
+ ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
1170
+ : creativesRes.paginationIncompleteReason || creativesRes.reason || null,
1171
+ },
1172
+ telemetry: computeRunEconomicsTelemetry({
1173
+ billableCount: reconciledItems.length,
1174
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1175
+ durationMs,
1176
+ requestCount: Math.max(1, totalRpcRequests),
1177
+ }),
1178
+ });
1179
+ }
1180
+
1181
+ const maxResultsCap = Math.min(200, Math.max(1, Number(input?.maxResults) || 40));
1182
+
1183
+ // Step 3: Strict Deduplication & Validation Gate before emitting Pay-per-Event charges
1184
+ const uniqueExtractedItems = [];
1185
+ const seenInRunIds = new Set();
1186
+
1187
+ for (const item of creativesRes.creatives) {
1188
+ if (
1189
+ !item ||
1190
+ !ADVERTISER_ID_RE.test(String(item.advertiserId || '')) ||
1191
+ !CREATIVE_ID_RE.test(String(item.creativeId || ''))
1192
+ ) {
1193
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1194
+ return finalizeActorReturn({
1195
+ ok: false,
1196
+ status: 'SCHEMA_DRIFT_HALT',
1197
+ reason: 'Normalized creative record failed AR.../CR... identifier gate prior to billing',
1198
+ billingMode,
1199
+ queryMeta: entityRes.queryMeta,
1200
+ selectedEntities: entityRes.selectedEntities,
1201
+ items: reconciledItems,
1202
+ extractedCount: 0,
1203
+ deliveredCount,
1204
+ billableCount: reconciledItems.length,
1205
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1206
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1207
+ chargeReceipts: [],
1208
+ pagination: {
1209
+ nextPageToken: null,
1210
+ paginationStatus: 'SCHEMA_DRIFT_HALT',
1211
+ paginationIncompleteReason: 'INVALID_CREATIVE_IDENTIFIER',
1212
+ },
1213
+ telemetry: computeRunEconomicsTelemetry({
1214
+ billableCount: reconciledItems.length,
1215
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1216
+ durationMs,
1217
+ requestCount: Math.max(1, totalRpcRequests),
1218
+ }),
1219
+ });
1220
+ }
1221
+
1222
+ if (!seenInRunIds.has(item.creativeId)) {
1223
+ seenInRunIds.add(item.creativeId);
1224
+ uniqueExtractedItems.push(item);
1225
+ if (uniqueExtractedItems.length >= maxResultsCap) {
1226
+ break;
1227
+ }
1228
+ }
1229
+ }
1230
+
1231
+ // Optional Step 3b: Enrich up to 10 creatives (default 5) via LookupService/GetCreativeById when includeCreativeDetails === true
1232
+ if (input?.includeCreativeDetails === true && uniqueExtractedItems.length > 0) {
1233
+ const requestedEnrichCap = Number(input?.maxEnrichItems ?? 5);
1234
+ const enrichCap = Math.min(
1235
+ 10,
1236
+ Math.max(1, Number.isFinite(requestedEnrichCap) ? Math.floor(requestedEnrichCap) : 5),
1237
+ uniqueExtractedItems.length
1238
+ );
1239
+ let rateLimitedInEnrich = false;
1240
+ for (let i = 0; i < enrichCap; i += 1) {
1241
+ const targetItem = uniqueExtractedItems[i];
1242
+ if (rateLimitedInEnrich) {
1243
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1244
+ 'NOT_ATTEMPTED_RATE_LIMITED'
1245
+ );
1246
+ continue;
1247
+ }
1248
+ if (nowMs() >= rpcDeadlineAtMs) {
1249
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1250
+ 'NOT_ATTEMPTED_DEADLINE'
1251
+ );
1252
+ continue;
1253
+ }
1254
+ const detailRes = await getCreativeDetailsById({
1255
+ advertiserId: targetItem.advertiserId,
1256
+ creativeId: targetItem.creativeId,
1257
+ includeAdCopy: Boolean(input?.includeAdCopy),
1258
+ fetchImpl: hooks.fetchImpl,
1259
+ nowFn: nowMs,
1260
+ sleepFn,
1261
+ randomFn,
1262
+ timeoutMs: input?.timeoutMs,
1263
+ maxRetries: input?.maxRetries !== undefined ? input.maxRetries : 1,
1264
+ deadlineAtMs: rpcDeadlineAtMs,
1265
+ });
1266
+ totalRpcRequests += detailRes.rpcAttempts;
1267
+ if (detailRes.status === 'UPSTREAM_RATE_LIMITED') {
1268
+ rateLimitedInEnrich = true;
1269
+ targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
1270
+ 'NOT_ATTEMPTED_RATE_LIMITED'
1271
+ );
1272
+ continue;
1273
+ }
1274
+ if (detailRes.ok && detailRes.detail) {
1275
+ targetItem.creativeDetail = detailRes.detail;
1276
+ if (detailRes.detail.untrustedAdContent) {
1277
+ targetItem.untrustedAdContent = detailRes.detail.untrustedAdContent;
1278
+ if (
1279
+ detailRes.detail.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
1280
+ detailRes.detail.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
1281
+ ) {
1282
+ rateLimitedInEnrich = true;
1283
+ }
1284
+ }
1285
+ }
1286
+ }
1287
+ }
1288
+
1289
+ const extractedCount = uniqueExtractedItems.length;
1290
+
1291
+ // Filter out items already charged, delivered, or marked unknown in prior checkpoint
1292
+ const newBillableCandidates = uniqueExtractedItems.filter(
1293
+ (item) =>
1294
+ !chargedSet.has(item.creativeId) &&
1295
+ !deliveredSet.has(item.creativeId) &&
1296
+ !unknownOutcomeSet.has(item.creativeId)
1297
+ );
1298
+
1299
+ if (newBillableCandidates.length === 0 && reconciledItems.length === 0) {
1300
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1301
+ return finalizeActorReturn({
1302
+ ok: true,
1303
+ status: 'EMPTY_RESULT',
1304
+ billingMode,
1305
+ queryMeta: entityRes.queryMeta,
1306
+ selectedEntities: entityRes.selectedEntities,
1307
+ items: [],
1308
+ extractedCount,
1309
+ deliveredCount,
1310
+ billableCount: 0,
1311
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1312
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1313
+ chargeReceipts: [],
1314
+ pagination: {
1315
+ nextPageToken: creativesRes.nextPageToken,
1316
+ paginationStatus: creativesRes.paginationStatus,
1317
+ paginationIncompleteReason: creativesRes.paginationIncompleteReason,
1318
+ pagesFetched: creativesRes.pagesFetched,
1319
+ },
1320
+ cursorResumeContext: creativesRes.cursorResumeContext,
1321
+ telemetry: computeRunEconomicsTelemetry({
1322
+ billableCount: 0,
1323
+ chargedEventCount: hasChargeHook ? chargedSet.size : 0,
1324
+ durationMs,
1325
+ requestCount: Math.max(1, totalRpcRequests),
1326
+ }),
1327
+ });
1328
+ }
1329
+
1330
+ // Step 4: Chunked Dataset Push (25 items/chunk) & Crash-Safe Event Charging
1331
+ const chunkSize = ACTOR_PRICING_CONTRACT.chunkSize;
1332
+ const chargeReceipts = [];
1333
+ const newlyDeliveredItems = [];
1334
+ let chargedEventCount = hasChargeHook ? chargedSet.size : 0;
1335
+ let finalStatus = 'OK';
1336
+ let finalOk = true;
1337
+ let haltReason = undefined;
1338
+ const runtimeWarnings = Array.isArray(creativesRes.warnings) ? [...creativesRes.warnings] : [];
1339
+ const persistCheckpointPostCharge = async () => {
1340
+ try {
1341
+ await persistCheckpoint();
1342
+ } catch (cpErr) {
1343
+ runtimeWarnings.push(
1344
+ `Post-charge checkpoint write warning: ${cpErr?.message || String(cpErr)}`
1345
+ );
1346
+ }
1347
+ };
1348
+
1349
+ for (let offset = 0; offset < newBillableCandidates.length; offset += chunkSize) {
1350
+ const effectiveChunkDeadlineMs = hasChargeHook
1351
+ ? deadlineAtMs - step4ReserveMs
1352
+ : deadlineAtMs;
1353
+ if (nowMs() >= effectiveChunkDeadlineMs) {
1354
+ finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
1355
+ finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
1356
+ haltReason = 'Run deadline reserve reached prior to charging next chunk; halted cleanly before billing';
1357
+ break;
1358
+ }
1359
+
1360
+ const unchargedChunk = newBillableCandidates
1361
+ .slice(offset, offset + chunkSize)
1362
+ .filter(
1363
+ (c) =>
1364
+ !chargedSet.has(c.creativeId) &&
1365
+ !deliveredSet.has(c.creativeId) &&
1366
+ !unknownOutcomeSet.has(c.creativeId)
1367
+ );
1368
+
1369
+ if (unchargedChunk.length === 0) {
1370
+ continue;
1371
+ }
1372
+
1373
+ const chunkIds = unchargedChunk.map((c) => c.creativeId);
1374
+ const idempotencyKey = computeChunkIdempotencyKey(runId, chunkIds);
1375
+
1376
+ if (!hasChargeHook) {
1377
+ // Local / MCP unbilled execution mode: deliver items with chargedEventCount = 0
1378
+ const pushRes = await pushDataWithRetry(unchargedChunk);
1379
+ if (!pushRes.ok) {
1380
+ await persistCheckpointPostCharge();
1381
+ finalOk = false;
1382
+ finalStatus = 'PARTIAL_BILLING_HALT';
1383
+ haltReason = `Dataset pushData failed in local mode: ${pushRes.error?.message || String(pushRes.error)}`;
1384
+ break;
1385
+ }
1386
+ for (const c of unchargedChunk) {
1387
+ deliveredSet.add(c.creativeId);
1388
+ newlyDeliveredItems.push(c);
1389
+ }
1390
+ deliveredCount = deliveredSet.size;
1391
+ await persistCheckpointPostCharge();
1392
+ continue;
1393
+ }
1394
+
1395
+ // Step 4.1: Persist inFlightIds and stagedItemsById BEFORE invoking chargeEvent
1396
+ inFlightSet = new Set(chunkIds);
1397
+ for (const c of unchargedChunk) {
1398
+ stagedItemsById[c.creativeId] = c;
1399
+ }
1400
+ try {
1401
+ await persistCheckpoint();
1402
+ } catch (preCpErr) {
1403
+ inFlightSet = new Set();
1404
+ finalOk = false;
1405
+ finalStatus = 'PARTIAL_BILLING_HALT';
1406
+ haltReason = `Pre-charge checkpoint write failed before billing: ${preCpErr?.message || String(preCpErr)}`;
1407
+ break;
1408
+ }
1409
+
1410
+ if (nowMs() >= effectiveChunkDeadlineMs) {
1411
+ inFlightSet = new Set();
1412
+ await persistCheckpointPostCharge();
1413
+ finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
1414
+ finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
1415
+ haltReason =
1416
+ 'Run deadline reserve reached immediately after pre-charge checkpoint write; cleared inFlightIds and halted before billing';
1417
+ break;
1418
+ }
1419
+
1420
+ const receipt = {
1421
+ idempotencyKey,
1422
+ eventName: ACTOR_PRICING_CONTRACT.eventName,
1423
+ count: unchargedChunk.length,
1424
+ unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
1425
+ chunkAmountUsdc: Number(
1426
+ (unchargedChunk.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
1427
+ ),
1428
+ creativeIds: chunkIds,
1429
+ };
1430
+
1431
+ let chargeResult;
1432
+ try {
1433
+ chargeResult = await hooks.chargeEvent(receipt);
1434
+ } catch (chargeErr) {
1435
+ for (const id of chunkIds) {
1436
+ unknownOutcomeSet.add(id);
1437
+ }
1438
+ inFlightSet = new Set();
1439
+ await persistCheckpointPostCharge();
1440
+ finalOk = false;
1441
+ finalStatus = 'PARTIAL_BILLING_HALT';
1442
+ haltReason = `chargeEvent threw error: ${chargeErr?.message || String(chargeErr)}`;
1443
+ break;
1444
+ }
1445
+
1446
+ // Step 4.2: Strict ChargeResult validation (never fallback to ?? unchargedChunk.length)
1447
+ const chargedCount = chargeResult?.chargedCount;
1448
+ const eventChargeLimitReached = Boolean(chargeResult?.eventChargeLimitReached);
1449
+
1450
+ if (
1451
+ !chargeResult ||
1452
+ typeof chargeResult !== 'object' ||
1453
+ !Number.isInteger(chargedCount) ||
1454
+ chargedCount < 0 ||
1455
+ chargedCount > unchargedChunk.length
1456
+ ) {
1457
+ for (const id of chunkIds) {
1458
+ unknownOutcomeSet.add(id);
1459
+ }
1460
+ inFlightSet = new Set();
1461
+ await persistCheckpointPostCharge();
1462
+ finalOk = false;
1463
+ finalStatus = 'PARTIAL_BILLING_HALT';
1464
+ haltReason =
1465
+ 'chargeEvent returned invalid or missing integer chargedCount (transitioned chunk to UNKNOWN_CHARGE_OUTCOME)';
1466
+ break;
1467
+ }
1468
+
1469
+ // Step 4.3: Branch on validated chargedCount
1470
+ if (chargedCount === unchargedChunk.length) {
1471
+ // Full chunk charged: persist chargedIds immediately BEFORE pushData (best-effort so KVS glitch never blocks delivery)
1472
+ for (const id of chunkIds) {
1473
+ chargedSet.add(id);
1474
+ }
1475
+ inFlightSet = new Set();
1476
+ chargedEventCount = chargedSet.size;
1477
+ await persistCheckpointPostCharge();
1478
+
1479
+ const pushRes = await pushDataWithRetry(unchargedChunk);
1480
+ if (!pushRes.ok) {
1481
+ chargeReceipts.push({
1482
+ ...receipt,
1483
+ chargedCount,
1484
+ eventChargeLimitReached,
1485
+ });
1486
+ finalOk = false;
1487
+ finalStatus = 'PARTIAL_BILLING_HALT';
1488
+ haltReason = `Dataset pushData failed after charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
1489
+ break;
1490
+ }
1491
+ for (const c of unchargedChunk) {
1492
+ deliveredSet.add(c.creativeId);
1493
+ newlyDeliveredItems.push(c);
1494
+ }
1495
+ deliveredCount = deliveredSet.size;
1496
+ await persistCheckpointPostCharge();
1497
+
1498
+ chargeReceipts.push({
1499
+ ...receipt,
1500
+ chargedCount,
1501
+ eventChargeLimitReached,
1502
+ });
1503
+
1504
+ if (eventChargeLimitReached) {
1505
+ finalStatus = 'CHARGE_LIMIT_REACHED';
1506
+ break;
1507
+ }
1508
+ } else if (chargedCount === 0) {
1509
+ inFlightSet = new Set();
1510
+ await persistCheckpointPostCharge();
1511
+ chargeReceipts.push({
1512
+ ...receipt,
1513
+ chargedCount: 0,
1514
+ eventChargeLimitReached,
1515
+ });
1516
+ if (eventChargeLimitReached) {
1517
+ finalOk = true;
1518
+ finalStatus = 'CHARGE_LIMIT_REACHED';
1519
+ } else {
1520
+ finalOk = false;
1521
+ finalStatus = 'PARTIAL_BILLING_HALT';
1522
+ haltReason = 'chargeEvent returned chargedCount = 0 without eventChargeLimitReached';
1523
+ }
1524
+ break;
1525
+ } else {
1526
+ // Partial chunk charge (0 < chargedCount < unchargedChunk.length)
1527
+ const explicitChargedIds = Array.isArray(chargeResult.chargedCreativeIds)
1528
+ ? chargeResult.chargedCreativeIds.filter((id) => chunkIds.includes(id))
1529
+ : null;
1530
+
1531
+ if (explicitChargedIds && explicitChargedIds.length === chargedCount) {
1532
+ for (const id of explicitChargedIds) {
1533
+ chargedSet.add(id);
1534
+ }
1535
+ inFlightSet = new Set();
1536
+ chargedEventCount = chargedSet.size;
1537
+ await persistCheckpointPostCharge();
1538
+
1539
+ const paidSubset = unchargedChunk.filter((c) => explicitChargedIds.includes(c.creativeId));
1540
+ const pushRes = await pushDataWithRetry(paidSubset);
1541
+ if (!pushRes.ok) {
1542
+ chargeReceipts.push({
1543
+ ...receipt,
1544
+ chargedCount,
1545
+ chargedCreativeIds: explicitChargedIds,
1546
+ eventChargeLimitReached,
1547
+ });
1548
+ finalOk = false;
1549
+ finalStatus = 'PARTIAL_BILLING_HALT';
1550
+ haltReason = `Dataset pushData failed after partial charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
1551
+ break;
1552
+ }
1553
+ for (const c of paidSubset) {
1554
+ deliveredSet.add(c.creativeId);
1555
+ newlyDeliveredItems.push(c);
1556
+ }
1557
+ deliveredCount = deliveredSet.size;
1558
+ await persistCheckpointPostCharge();
1559
+
1560
+ chargeReceipts.push({
1561
+ ...receipt,
1562
+ chargedCount,
1563
+ chargedCreativeIds: explicitChargedIds,
1564
+ eventChargeLimitReached,
1565
+ });
1566
+
1567
+ finalOk = eventChargeLimitReached;
1568
+ finalStatus = eventChargeLimitReached ? 'CHARGE_LIMIT_REACHED' : 'PARTIAL_BILLING_HALT';
1569
+ if (!eventChargeLimitReached) {
1570
+ haltReason = `Partial chunk charge (${chargedCount}/${unchargedChunk.length}) without charge limit flag`;
1571
+ }
1572
+ break;
1573
+ } else if (eventChargeLimitReached) {
1574
+ // Standard Apify SDK ChargeResult when hitting user's max charge ceiling:
1575
+ // Attribute charge deterministically to the ordered prefix `unchargedChunk.slice(0, chargedCount)`
1576
+ // so the customer receives every item they paid for, and leave the remaining tail uncharged.
1577
+ const paidPrefix = unchargedChunk.slice(0, chargedCount);
1578
+ const paidPrefixIds = paidPrefix.map((c) => c.creativeId);
1579
+ for (const id of paidPrefixIds) {
1580
+ chargedSet.add(id);
1581
+ }
1582
+ inFlightSet = new Set();
1583
+ chargedEventCount = chargedSet.size;
1584
+ await persistCheckpointPostCharge();
1585
+
1586
+ const pushRes = await pushDataWithRetry(paidPrefix);
1587
+ if (!pushRes.ok) {
1588
+ chargeReceipts.push({
1589
+ ...receipt,
1590
+ chargedCount,
1591
+ chargedCreativeIds: paidPrefixIds,
1592
+ eventChargeLimitReached: true,
1593
+ });
1594
+ finalOk = false;
1595
+ finalStatus = 'PARTIAL_BILLING_HALT';
1596
+ haltReason = `Dataset pushData failed after charge limit prefix (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
1597
+ break;
1598
+ }
1599
+ for (const c of paidPrefix) {
1600
+ deliveredSet.add(c.creativeId);
1601
+ newlyDeliveredItems.push(c);
1602
+ }
1603
+ deliveredCount = deliveredSet.size;
1604
+ await persistCheckpointPostCharge();
1605
+
1606
+ chargeReceipts.push({
1607
+ ...receipt,
1608
+ chargedCount,
1609
+ chargedCreativeIds: paidPrefixIds,
1610
+ eventChargeLimitReached: true,
1611
+ });
1612
+
1613
+ finalOk = true;
1614
+ finalStatus = 'CHARGE_LIMIT_REACHED';
1615
+ break;
1616
+ } else {
1617
+ // Unexpected partial count without eventChargeLimitReached and without explicit IDs ->
1618
+ // move chunk to unknownOutcomeIds and halt fail-closed
1619
+ for (const id of chunkIds) {
1620
+ unknownOutcomeSet.add(id);
1621
+ }
1622
+ inFlightSet = new Set();
1623
+ chargedEventCount = chargedSet.size + chargedCount;
1624
+ await persistCheckpointPostCharge();
1625
+
1626
+ chargeReceipts.push({
1627
+ ...receipt,
1628
+ chargedCount,
1629
+ eventChargeLimitReached: false,
1630
+ });
1631
+
1632
+ finalOk = false;
1633
+ finalStatus = 'PARTIAL_BILLING_HALT';
1634
+ haltReason = `Ambiguous partial chunk charge (${chargedCount}/${unchargedChunk.length}) without chargedCreativeIds attribution or eventChargeLimitReached`;
1635
+ break;
1636
+ }
1637
+ }
1638
+ }
1639
+
1640
+ // Step 5: Rewind pagination cursor if billing halted or reached charge limit before delivering all extracted items
1641
+ let effectiveNextPageToken = creativesRes.nextPageToken;
1642
+ let effectivePaginationStatus = creativesRes.paginationStatus;
1643
+ let effectiveIncompleteReason = creativesRes.paginationIncompleteReason;
1644
+ const undeliveredCount = Math.max(0, newBillableCandidates.length - newlyDeliveredItems.length);
1645
+
1646
+ if (undeliveredCount > 0) {
1647
+ const firstUndeliveredItem = newBillableCandidates.find((c) => !deliveredSet.has(c.creativeId));
1648
+ const firstUndeliveredExtractedIdx = Array.isArray(creativesRes.creatives)
1649
+ ? creativesRes.creatives.findIndex((c) => c.creativeId === firstUndeliveredItem?.creativeId)
1650
+ : -1;
1651
+ const effectiveDeliveredPrefixLen =
1652
+ firstUndeliveredExtractedIdx >= 0 ? firstUndeliveredExtractedIdx : newlyDeliveredItems.length;
1653
+ effectiveNextPageToken = computeRewoundCursorForDeliveredPrefix(
1654
+ creativesRes.cursorResumeContext,
1655
+ effectiveDeliveredPrefixLen,
1656
+ creativesRes.nextPageToken
1657
+ );
1658
+ effectivePaginationStatus =
1659
+ finalStatus === 'CHARGE_LIMIT_REACHED'
1660
+ ? 'CHARGE_LIMIT_REACHED_HAS_MORE'
1661
+ : finalStatus === 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
1662
+ ? 'DEADLINE_EXCEEDED_HAS_MORE'
1663
+ : 'PARTIAL_BILLING_HALT';
1664
+ effectiveIncompleteReason = finalStatus;
1665
+ }
1666
+
1667
+ const paginationMeta = {
1668
+ nextPageToken: effectiveNextPageToken,
1669
+ paginationStatus: effectivePaginationStatus,
1670
+ paginationIncompleteReason: effectiveIncompleteReason,
1671
+ pagesFetched: creativesRes.pagesFetched,
1672
+ ...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
1673
+ ...(undeliveredCount > 0 ? { undeliveredCount } : {}),
1674
+ };
1675
+
1676
+ // Merge reconciled items + newly delivered items with strict creativeId deduplication
1677
+ const allOutputItems = [];
1678
+ const outputSeenIds = new Set();
1679
+ for (const item of [...reconciledItems, ...newlyDeliveredItems]) {
1680
+ if (item?.creativeId && !outputSeenIds.has(item.creativeId)) {
1681
+ outputSeenIds.add(item.creativeId);
1682
+ allOutputItems.push(item);
1683
+ }
1684
+ }
1685
+
1686
+ const simulatedBillableCount = hasChargeHook ? chargedEventCount : allOutputItems.length;
1687
+ const durationMs = Math.max(1, nowMs() - startedAtMs);
1688
+ const telemetry = computeRunEconomicsTelemetry({
1689
+ billableCount: simulatedBillableCount,
1690
+ chargedEventCount,
1691
+ durationMs,
1692
+ requestCount: Math.max(1, totalRpcRequests),
1693
+ });
1694
+
1695
+ return finalizeActorReturn({
1696
+ ok: finalOk,
1697
+ status: finalStatus,
1698
+ reason: haltReason,
1699
+ billingMode,
1700
+ queryMeta: entityRes.queryMeta,
1701
+ selectedEntities: entityRes.selectedEntities,
1702
+ items: allOutputItems,
1703
+ extractedCount,
1704
+ deliveredCount,
1705
+ billableCount: simulatedBillableCount,
1706
+ chargedEventCount,
1707
+ unknownOutcomeIds: Array.from(unknownOutcomeSet),
1708
+ chargeReceipts,
1709
+ pagination: paginationMeta,
1710
+ ...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
1711
+ cursorResumeContext: creativesRes.cursorResumeContext,
1712
+ telemetry,
1713
+ });
1714
+ }