@revclamp/google-ads-transparency-mcp 0.0.0-stage → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.actor/actor.json +78 -0
- package/.actor/input_schema.json +79 -0
- package/.well-known/glama.json +6 -0
- package/LICENSE +21 -0
- package/README.md +134 -2
- package/actor_config.json +29 -0
- package/assets/icon-180.png +0 -0
- package/assets/icon-32.png +0 -0
- package/assets/icon.svg +26 -0
- package/assets/playground.css +468 -0
- package/assets/playground.js +817 -0
- package/glama.json +6 -0
- package/index.mjs +316 -0
- package/package.json +55 -4
- package/server.json +45 -0
- package/src/apify_actor.mjs +2208 -0
- package/src/egress_guard.mjs +102 -0
- package/src/mcp_server.mjs +1171 -0
- package/src/rpc_client.mjs +3925 -0
- package/src/web_preview_server.mjs +420 -0
|
@@ -0,0 +1,2208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Apify Pay-per-Event Execution, Native Platform Adapter & Crash-Safe Billing State Machine (src/apify_actor.mjs)
|
|
3
|
+
*
|
|
4
|
+
* Enforces deterministic Pay-per-Event billing and delivery invariants:
|
|
5
|
+
* 1. Explicit Accounting Separation: Tracks `extractedCount`, `deliveredCount`,
|
|
6
|
+
* `billableCount` (simulated unit-economics count), and confirmed `chargedEventCount`
|
|
7
|
+
* (`0` in local/MCP unbilled mode; strictly validated integer in Apify billing mode).
|
|
8
|
+
* 2. Native Zero-Dependency Apify Platform Adapter (`createApifyPlatformHooks` & `loadApifyActorInput`):
|
|
9
|
+
* Wires Apify Key-Value Store (`INPUT` & `ADPULSE_BILLING_CHECKPOINT`), Dataset (`pushData`),
|
|
10
|
+
* and Pay-per-Event Charging (`POST /v2/actor-runs/:runId/charge` with `Idempotency-Key`)
|
|
11
|
+
* when running on Apify (`APIFY_IS_AT_HOME=1` / `APIFY_TOKEN` + `APIFY_ACTOR_RUN_ID`) or local storage.
|
|
12
|
+
* 3. Crash-Safe KVS Checkpoint State Machine (`inFlightIds` -> `chargedIds` -> `deliveredIds`
|
|
13
|
+
* or `unknownOutcomeIds`), deterministic prefix delivery when `eventChargeLimitReached` returns
|
|
14
|
+
* partial `chargedCount`, cursor rewind on mid-stream halt/limit, and honest zero-event telemetry.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import crypto from 'node:crypto';
|
|
20
|
+
import {
|
|
21
|
+
resolveAdvertiserEntities,
|
|
22
|
+
fetchAdvertiserCreatives,
|
|
23
|
+
getCreativeDetailsById,
|
|
24
|
+
createEmptyUntrustedAdContent,
|
|
25
|
+
computeRewoundCursorForDeliveredPrefix,
|
|
26
|
+
verifyEgressUrlSsrfSafe,
|
|
27
|
+
CREATIVE_ID_RE,
|
|
28
|
+
ADVERTISER_ID_RE,
|
|
29
|
+
DEFAULT_RUN_DEADLINE_MS,
|
|
30
|
+
} from './rpc_client.mjs';
|
|
31
|
+
|
|
32
|
+
export const ACTOR_PRICING_CONTRACT = Object.freeze({
|
|
33
|
+
eventName: 'ad-creative-scraped',
|
|
34
|
+
actorStartFeeUsdc: 0.001,
|
|
35
|
+
priceUsdc: 0.0012,
|
|
36
|
+
costCeilingUsdc: 0.0001,
|
|
37
|
+
platformFeeRate: 0.2,
|
|
38
|
+
netPriceUsdc: 0.00096,
|
|
39
|
+
minMarginMultiplier: 2.5,
|
|
40
|
+
computeRatePerGbHourUsdc: 0.3,
|
|
41
|
+
maxMemoryMb: 256,
|
|
42
|
+
maxTimeoutSec: 25,
|
|
43
|
+
chunkSize: 25,
|
|
44
|
+
checkpointKey: 'ADPULSE_BILLING_CHECKPOINT',
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Computes content-addressed batch idempotency key (`sha256(runId + '\x1f' + sortedChunkIds)`).
|
|
49
|
+
*
|
|
50
|
+
* @param {string} runId
|
|
51
|
+
* @param {string[]} creativeIds
|
|
52
|
+
* @returns {string}
|
|
53
|
+
*/
|
|
54
|
+
export function computeChunkIdempotencyKey(runId, creativeIds) {
|
|
55
|
+
const safeRunId = String(runId || 'default_run');
|
|
56
|
+
const sortedIds = [...creativeIds].map((id) => String(id)).sort();
|
|
57
|
+
return crypto
|
|
58
|
+
.createHash('sha256')
|
|
59
|
+
.update(`${safeRunId}\x1f${sortedIds.join('\x1f')}`, 'utf8')
|
|
60
|
+
.digest('hex');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Computes unit-economics and margin telemetry for a single Actor/MCP execution run.
|
|
65
|
+
* Uses Apify's standard `$0.30 / GB-hour` compute rate (`0.25 GB` at `256MB`).
|
|
66
|
+
* When `billableCount === 0`, returns `economicsStatus: 'NO_BILLABLE_EVENTS'` with
|
|
67
|
+
* `withinCostCeiling: false` and `marginPositive: false`.
|
|
68
|
+
*
|
|
69
|
+
* @param {object} params
|
|
70
|
+
* @param {number} params.billableCount
|
|
71
|
+
* @param {number} [params.chargedEventCount=0]
|
|
72
|
+
* @param {number} params.durationMs
|
|
73
|
+
* @param {number} [params.memoryMb=256]
|
|
74
|
+
* @param {number} [params.requestCount=1]
|
|
75
|
+
* @returns {object}
|
|
76
|
+
*/
|
|
77
|
+
export function computeRunEconomicsTelemetry({
|
|
78
|
+
billableCount = 0,
|
|
79
|
+
chargedEventCount = 0,
|
|
80
|
+
durationMs = 100,
|
|
81
|
+
memoryMb =
|
|
82
|
+
Number(process.env.ACTOR_MEMORY_MBYTES || process.env.APIFY_MEMORY_MBYTES) ||
|
|
83
|
+
ACTOR_PRICING_CONTRACT.maxMemoryMb,
|
|
84
|
+
requestCount = 1,
|
|
85
|
+
unitPriceUsdc = ACTOR_PRICING_CONTRACT.priceUsdc,
|
|
86
|
+
actorStartFeeUsdc = ACTOR_PRICING_CONTRACT.actorStartFeeUsdc,
|
|
87
|
+
} = {}) {
|
|
88
|
+
const safeCount = Math.max(0, Math.floor(Number(billableCount) || 0));
|
|
89
|
+
const safeChargedCount = Math.max(0, Math.floor(Number(chargedEventCount) || 0));
|
|
90
|
+
const safeDurationMs = Math.max(1, Number(durationMs) || 1);
|
|
91
|
+
const safeMemoryMb = Math.min(1024, Math.max(64, Number(memoryMb) || 256));
|
|
92
|
+
const safeRequests = Math.max(1, Math.floor(Number(requestCount) || 1));
|
|
93
|
+
const resolvedUnitPriceUsdc =
|
|
94
|
+
Number.isFinite(Number(unitPriceUsdc)) && Number(unitPriceUsdc) > 0
|
|
95
|
+
? Number(unitPriceUsdc)
|
|
96
|
+
: ACTOR_PRICING_CONTRACT.priceUsdc;
|
|
97
|
+
const resolvedNetPriceUsdc = Number(
|
|
98
|
+
(resolvedUnitPriceUsdc * (1 - ACTOR_PRICING_CONTRACT.platformFeeRate)).toFixed(6)
|
|
99
|
+
);
|
|
100
|
+
const resolvedStartFeeUsdc =
|
|
101
|
+
Number.isFinite(Number(actorStartFeeUsdc)) && Number(actorStartFeeUsdc) >= 0
|
|
102
|
+
? Number(actorStartFeeUsdc)
|
|
103
|
+
: ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
|
|
104
|
+
|
|
105
|
+
// Apify compute rate: $0.30 per GB-hour; 256MB = 0.25 GB -> $0.075/hour -> $0.00002083/sec
|
|
106
|
+
const computeHours = safeDurationMs / 3600000;
|
|
107
|
+
const computeCostUsdc =
|
|
108
|
+
(safeMemoryMb / 1024) * computeHours * ACTOR_PRICING_CONTRACT.computeRatePerGbHourUsdc;
|
|
109
|
+
const networkOverheadUsdc = safeRequests * 0.0000002;
|
|
110
|
+
const totalRunCostUsdc = Number((computeCostUsdc + networkOverheadUsdc).toFixed(8));
|
|
111
|
+
|
|
112
|
+
const configuredFloorMultiplier = Number(
|
|
113
|
+
(resolvedNetPriceUsdc / ACTOR_PRICING_CONTRACT.costCeilingUsdc).toFixed(2)
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
if (safeCount === 0) {
|
|
117
|
+
return {
|
|
118
|
+
economicsStatus: 'NO_BILLABLE_EVENTS',
|
|
119
|
+
billableCount: 0,
|
|
120
|
+
chargedEventCount: safeChargedCount,
|
|
121
|
+
durationMs: safeDurationMs,
|
|
122
|
+
memoryMb: safeMemoryMb,
|
|
123
|
+
requestCount: safeRequests,
|
|
124
|
+
actorStartFeeUsdc: resolvedStartFeeUsdc,
|
|
125
|
+
totalRunCostUsdc,
|
|
126
|
+
actualCostPerEventUsdc: 0,
|
|
127
|
+
costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
|
|
128
|
+
withinCostCeiling: false,
|
|
129
|
+
grossRevenueUsdc: 0,
|
|
130
|
+
netRevenueUsdc: 0,
|
|
131
|
+
configuredFloorMultiplier,
|
|
132
|
+
effectiveMarginMultiplier: 0,
|
|
133
|
+
marginPositive: false,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const actualCostPerEventUsdc = Number((totalRunCostUsdc / safeCount).toFixed(8));
|
|
138
|
+
const grossRevenueUsdc = Number((safeCount * resolvedUnitPriceUsdc).toFixed(6));
|
|
139
|
+
const netRevenueUsdc = Number((safeCount * resolvedNetPriceUsdc).toFixed(6));
|
|
140
|
+
const effectiveMarginMultiplier =
|
|
141
|
+
actualCostPerEventUsdc > 0
|
|
142
|
+
? Number((resolvedNetPriceUsdc / actualCostPerEventUsdc).toFixed(2))
|
|
143
|
+
: configuredFloorMultiplier;
|
|
144
|
+
|
|
145
|
+
const withinCostCeiling = actualCostPerEventUsdc <= ACTOR_PRICING_CONTRACT.costCeilingUsdc;
|
|
146
|
+
const marginPositive =
|
|
147
|
+
withinCostCeiling &&
|
|
148
|
+
configuredFloorMultiplier >= ACTOR_PRICING_CONTRACT.minMarginMultiplier;
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
economicsStatus: marginPositive ? 'PROFITABLE' : 'COST_CEILING_EXCEEDED',
|
|
152
|
+
billableCount: safeCount,
|
|
153
|
+
chargedEventCount: safeChargedCount,
|
|
154
|
+
durationMs: safeDurationMs,
|
|
155
|
+
memoryMb: safeMemoryMb,
|
|
156
|
+
requestCount: safeRequests,
|
|
157
|
+
actorStartFeeUsdc: resolvedStartFeeUsdc,
|
|
158
|
+
totalRunCostUsdc,
|
|
159
|
+
actualCostPerEventUsdc,
|
|
160
|
+
costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
|
|
161
|
+
withinCostCeiling,
|
|
162
|
+
grossRevenueUsdc,
|
|
163
|
+
netRevenueUsdc,
|
|
164
|
+
configuredFloorMultiplier,
|
|
165
|
+
effectiveMarginMultiplier,
|
|
166
|
+
marginPositive,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Validates that an Apify API base URL is safe (`https://api.apify.com` by default).
|
|
172
|
+
* When `rawApiBaseUrl` is an internal HTTP endpoint on Apify Cloud and a valid HTTPS
|
|
173
|
+
* `fallbackPublicBaseUrl` (`APIFY_API_PUBLIC_BASE_URL`) is provided, uses the HTTPS public URL.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} [rawApiBaseUrl]
|
|
176
|
+
* @param {string} [fallbackPublicBaseUrl]
|
|
177
|
+
* @returns {string}
|
|
178
|
+
*/
|
|
179
|
+
function resolveSafeApifyApiBaseUrl(rawApiBaseUrl, fallbackPublicBaseUrl = undefined) {
|
|
180
|
+
const candidate = String(rawApiBaseUrl || fallbackPublicBaseUrl || 'https://api.apify.com')
|
|
181
|
+
.trim()
|
|
182
|
+
.replace(/\/+$/, '');
|
|
183
|
+
const ssrf = verifyEgressUrlSsrfSafe(candidate);
|
|
184
|
+
if (!ssrf.safe) {
|
|
185
|
+
if (fallbackPublicBaseUrl) {
|
|
186
|
+
const publicCandidate = String(fallbackPublicBaseUrl).trim().replace(/\/+$/, '');
|
|
187
|
+
const publicSsrf = verifyEgressUrlSsrfSafe(publicCandidate);
|
|
188
|
+
if (publicSsrf.safe) {
|
|
189
|
+
return publicCandidate;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
throw new Error(`Invalid or unsafe APIFY_API_BASE_URL '${candidate}': ${ssrf.reason}`);
|
|
193
|
+
}
|
|
194
|
+
return candidate;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Loads Actor input JSON from CLI `--input-json`, `APIFY_INPUT_PATH`, local Apify storage
|
|
199
|
+
* (`$ACTOR_STORAGE_DIR` / `$APIFY_LOCAL_STORAGE_DIR/key_value_stores/default/INPUT.json`),
|
|
200
|
+
* or Apify Platform KVS REST API.
|
|
201
|
+
*
|
|
202
|
+
* @param {string[]} [cliArgs=[]]
|
|
203
|
+
* @param {object} [env=process.env]
|
|
204
|
+
* @param {Function} [fetchImpl=globalThis.fetch]
|
|
205
|
+
* @returns {Promise<object>}
|
|
206
|
+
*/
|
|
207
|
+
export async function loadApifyActorInput(
|
|
208
|
+
cliArgs = [],
|
|
209
|
+
env = process.env,
|
|
210
|
+
fetchImpl = globalThis.fetch
|
|
211
|
+
) {
|
|
212
|
+
const inputIdx = Array.isArray(cliArgs) ? cliArgs.indexOf('--input-json') : -1;
|
|
213
|
+
if (inputIdx !== -1 && cliArgs[inputIdx + 1]) {
|
|
214
|
+
const raw = fs.readFileSync(cliArgs[inputIdx + 1], 'utf8');
|
|
215
|
+
return JSON.parse(raw);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
|
|
219
|
+
const applyAtHomeSchemaDefaults = (parsedInput) => {
|
|
220
|
+
if (
|
|
221
|
+
isAtHome &&
|
|
222
|
+
parsedInput &&
|
|
223
|
+
typeof parsedInput === 'object' &&
|
|
224
|
+
!Array.isArray(parsedInput) &&
|
|
225
|
+
!Object.prototype.hasOwnProperty.call(parsedInput, 'query') &&
|
|
226
|
+
!Object.prototype.hasOwnProperty.call(parsedInput, 'advertiserId') &&
|
|
227
|
+
!Object.prototype.hasOwnProperty.call(parsedInput, 'pageToken')
|
|
228
|
+
) {
|
|
229
|
+
return { query: 'Stripe', ...parsedInput };
|
|
230
|
+
}
|
|
231
|
+
return parsedInput;
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
if (env?.APIFY_INPUT_PATH && fs.existsSync(env.APIFY_INPUT_PATH)) {
|
|
235
|
+
const raw = fs.readFileSync(env.APIFY_INPUT_PATH, 'utf8');
|
|
236
|
+
return applyAtHomeSchemaDefaults(raw && raw.trim() ? JSON.parse(raw) : {});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const localStorageDir =
|
|
240
|
+
env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
|
|
241
|
+
if (localStorageDir) {
|
|
242
|
+
const kvsId =
|
|
243
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ||
|
|
244
|
+
env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID ||
|
|
245
|
+
'default';
|
|
246
|
+
const inputKey = env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT';
|
|
247
|
+
for (const candidateName of [`${inputKey}.json`, inputKey]) {
|
|
248
|
+
const localInputPath = path.join(
|
|
249
|
+
localStorageDir,
|
|
250
|
+
'key_value_stores',
|
|
251
|
+
kvsId,
|
|
252
|
+
candidateName
|
|
253
|
+
);
|
|
254
|
+
if (fs.existsSync(localInputPath)) {
|
|
255
|
+
const raw = fs.readFileSync(localInputPath, 'utf8');
|
|
256
|
+
return applyAtHomeSchemaDefaults(raw && raw.trim() ? JSON.parse(raw) : {});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const kvsId =
|
|
262
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
|
|
263
|
+
const token = env?.APIFY_TOKEN;
|
|
264
|
+
if (kvsId && token && typeof fetchImpl === 'function') {
|
|
265
|
+
const apiBase = resolveSafeApifyApiBaseUrl(
|
|
266
|
+
env?.APIFY_API_BASE_URL,
|
|
267
|
+
env?.APIFY_API_PUBLIC_BASE_URL
|
|
268
|
+
);
|
|
269
|
+
const inputKey = encodeURIComponent(
|
|
270
|
+
env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT'
|
|
271
|
+
);
|
|
272
|
+
const url = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${inputKey}`;
|
|
273
|
+
const controller = new AbortController();
|
|
274
|
+
const timer = setTimeout(() => controller.abort(), 4000);
|
|
275
|
+
try {
|
|
276
|
+
const res = await fetchImpl(url, {
|
|
277
|
+
method: 'GET',
|
|
278
|
+
headers: {
|
|
279
|
+
authorization: `Bearer ${token}`,
|
|
280
|
+
accept: 'application/json',
|
|
281
|
+
},
|
|
282
|
+
signal: controller.signal,
|
|
283
|
+
});
|
|
284
|
+
if (res.status === 200) {
|
|
285
|
+
const text = await res.text();
|
|
286
|
+
const parsed = text && text.trim() ? JSON.parse(text) : {};
|
|
287
|
+
return applyAtHomeSchemaDefaults(parsed);
|
|
288
|
+
}
|
|
289
|
+
if (res.status === 204 || res.status === 404) {
|
|
290
|
+
return applyAtHomeSchemaDefaults({});
|
|
291
|
+
}
|
|
292
|
+
return {};
|
|
293
|
+
} finally {
|
|
294
|
+
clearTimeout(timer);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
return applyAtHomeSchemaDefaults({});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Creates native zero-dependency Apify Platform runtime hooks (`runId`, `isAtHome`,
|
|
303
|
+
* `loadCheckpoint`, `saveCheckpoint`, `listDeliveredIds`, `pushData`, `chargeEvent`, `saveOutput`)
|
|
304
|
+
* when Apify environment variables (`ACTOR_*` primary, `APIFY_*` fallback) or local
|
|
305
|
+
* storage directories are configured. Returns `{}` in pure local CLI/MCP mode.
|
|
306
|
+
*
|
|
307
|
+
* @param {object} [env=process.env]
|
|
308
|
+
* @param {Function} [fetchImpl=globalThis.fetch]
|
|
309
|
+
* @returns {object}
|
|
310
|
+
*/
|
|
311
|
+
export function createApifyPlatformHooks(env = process.env, fetchImpl = globalThis.fetch) {
|
|
312
|
+
const rawRunId = env?.ACTOR_RUN_ID || env?.APIFY_ACTOR_RUN_ID;
|
|
313
|
+
const runId = rawRunId ? String(rawRunId).trim() : undefined;
|
|
314
|
+
const token = env?.APIFY_TOKEN ? String(env.APIFY_TOKEN).trim() : undefined;
|
|
315
|
+
const rawDatasetId = env?.ACTOR_DEFAULT_DATASET_ID || env?.APIFY_DEFAULT_DATASET_ID;
|
|
316
|
+
const datasetId = rawDatasetId ? String(rawDatasetId).trim() : undefined;
|
|
317
|
+
const rawKvsId =
|
|
318
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
|
|
319
|
+
const kvsId = rawKvsId ? String(rawKvsId).trim() : undefined;
|
|
320
|
+
const localStorageDir =
|
|
321
|
+
env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
|
|
322
|
+
const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
|
|
323
|
+
|
|
324
|
+
const hasCloudApi = Boolean((isAtHome || runId) && token && typeof fetchImpl === 'function');
|
|
325
|
+
if (!hasCloudApi && !localStorageDir) {
|
|
326
|
+
if (isAtHome) {
|
|
327
|
+
return { isAtHome: true, ...(runId ? { runId } : {}) };
|
|
328
|
+
}
|
|
329
|
+
return runId ? { runId } : {};
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const hooks = {};
|
|
333
|
+
if (runId) {
|
|
334
|
+
hooks.runId = runId;
|
|
335
|
+
}
|
|
336
|
+
if (isAtHome || hasCloudApi) {
|
|
337
|
+
hooks.isAtHome = true;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (hasCloudApi) {
|
|
341
|
+
const apiBase = resolveSafeApifyApiBaseUrl(
|
|
342
|
+
env?.APIFY_API_BASE_URL,
|
|
343
|
+
env?.APIFY_API_PUBLIC_BASE_URL
|
|
344
|
+
);
|
|
345
|
+
let pricingInfoChargeEvents = null;
|
|
346
|
+
const rawPricingInfoEnv = env?.APIFY_ACTOR_PRICING_INFO || env?.ACTOR_PRICING_INFO;
|
|
347
|
+
if (rawPricingInfoEnv) {
|
|
348
|
+
try {
|
|
349
|
+
const parsedPricing = JSON.parse(String(rawPricingInfoEnv));
|
|
350
|
+
const eventsObj = parsedPricing?.pricingPerEvent?.actorChargeEvents;
|
|
351
|
+
if (eventsObj && typeof eventsObj === 'object') {
|
|
352
|
+
pricingInfoChargeEvents = eventsObj;
|
|
353
|
+
}
|
|
354
|
+
} catch {
|
|
355
|
+
pricingInfoChargeEvents = null;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
const rawMaxChargeStr =
|
|
359
|
+
env?.ACTOR_MAX_TOTAL_CHARGE_USD !== undefined && env?.ACTOR_MAX_TOTAL_CHARGE_USD !== null
|
|
360
|
+
? String(env.ACTOR_MAX_TOTAL_CHARGE_USD).trim()
|
|
361
|
+
: '';
|
|
362
|
+
const maxTotalChargeUsdRaw = rawMaxChargeStr !== '' ? Number(rawMaxChargeStr) : NaN;
|
|
363
|
+
const hasMaxChargeUsd = Number.isFinite(maxTotalChargeUsdRaw) && maxTotalChargeUsdRaw >= 0;
|
|
364
|
+
const includeStartFeeFlag = String(env?.ACTOR_INCLUDE_START_FEE ?? '1')
|
|
365
|
+
.trim()
|
|
366
|
+
.toLowerCase();
|
|
367
|
+
const includeStartFee = includeStartFeeFlag !== '0' && includeStartFeeFlag !== 'false';
|
|
368
|
+
const pricingInfoStartFeeUsd = Number(
|
|
369
|
+
pricingInfoChargeEvents?.['apify-actor-start']?.eventPriceUsd
|
|
370
|
+
);
|
|
371
|
+
const explicitStartFeeUsd =
|
|
372
|
+
env?.ACTOR_START_FEE_USD !== undefined &&
|
|
373
|
+
env?.ACTOR_START_FEE_USD !== null &&
|
|
374
|
+
String(env.ACTOR_START_FEE_USD).trim() !== ''
|
|
375
|
+
? Number(env.ACTOR_START_FEE_USD)
|
|
376
|
+
: Number.isFinite(pricingInfoStartFeeUsd) && pricingInfoStartFeeUsd >= 0
|
|
377
|
+
? pricingInfoStartFeeUsd
|
|
378
|
+
: ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
|
|
379
|
+
const startFeeMicros =
|
|
380
|
+
includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
|
|
381
|
+
? Math.round(explicitStartFeeUsd * 1_000_000)
|
|
382
|
+
: 0;
|
|
383
|
+
const maxBudgetMicros = hasMaxChargeUsd ? Math.round(maxTotalChargeUsdRaw * 1_000_000) : 0;
|
|
384
|
+
let cumulativeChargedInAdapter = 0;
|
|
385
|
+
let adapterChargeLimitLatched = false;
|
|
386
|
+
hooks.actorStartFeeUsdc =
|
|
387
|
+
includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
|
|
388
|
+
? explicitStartFeeUsd
|
|
389
|
+
: 0;
|
|
390
|
+
|
|
391
|
+
if (kvsId) {
|
|
392
|
+
const checkpointRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${ACTOR_PRICING_CONTRACT.checkpointKey}`;
|
|
393
|
+
const outputRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/OUTPUT`;
|
|
394
|
+
|
|
395
|
+
hooks.loadCheckpoint = async () => {
|
|
396
|
+
const controller = new AbortController();
|
|
397
|
+
const timer = setTimeout(() => controller.abort(), 1500);
|
|
398
|
+
try {
|
|
399
|
+
const res = await fetchImpl(checkpointRecordUrl, {
|
|
400
|
+
method: 'GET',
|
|
401
|
+
headers: {
|
|
402
|
+
authorization: `Bearer ${token}`,
|
|
403
|
+
accept: 'application/json',
|
|
404
|
+
},
|
|
405
|
+
signal: controller.signal,
|
|
406
|
+
});
|
|
407
|
+
if (res.status === 404 || res.status === 204) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
if (res.status >= 200 && res.status < 300) {
|
|
411
|
+
const text = await res.text();
|
|
412
|
+
const loaded = text && text.trim() ? JSON.parse(text) : null;
|
|
413
|
+
if (
|
|
414
|
+
loaded &&
|
|
415
|
+
Array.isArray(loaded.chargedIds) &&
|
|
416
|
+
(!loaded.runId || !runId || String(loaded.runId) === String(runId))
|
|
417
|
+
) {
|
|
418
|
+
cumulativeChargedInAdapter = loaded.chargedIds.length;
|
|
419
|
+
}
|
|
420
|
+
return loaded;
|
|
421
|
+
}
|
|
422
|
+
throw new Error(`Apify KVS loadCheckpoint failed with HTTP ${res.status}`);
|
|
423
|
+
} finally {
|
|
424
|
+
clearTimeout(timer);
|
|
425
|
+
}
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
hooks.saveCheckpoint = async (snapshot) => {
|
|
429
|
+
const controller = new AbortController();
|
|
430
|
+
const timer = setTimeout(() => controller.abort(), 1500);
|
|
431
|
+
try {
|
|
432
|
+
const res = await fetchImpl(checkpointRecordUrl, {
|
|
433
|
+
method: 'PUT',
|
|
434
|
+
headers: {
|
|
435
|
+
authorization: `Bearer ${token}`,
|
|
436
|
+
'content-type': 'application/json; charset=utf-8',
|
|
437
|
+
},
|
|
438
|
+
body: JSON.stringify(snapshot),
|
|
439
|
+
signal: controller.signal,
|
|
440
|
+
});
|
|
441
|
+
if (res.status < 200 || res.status >= 300) {
|
|
442
|
+
throw new Error(`Apify KVS saveCheckpoint HTTP ${res.status}`);
|
|
443
|
+
}
|
|
444
|
+
} finally {
|
|
445
|
+
clearTimeout(timer);
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
hooks.saveOutput = async (outputRecord) => {
|
|
450
|
+
const controller = new AbortController();
|
|
451
|
+
const timer = setTimeout(() => controller.abort(), 1500);
|
|
452
|
+
try {
|
|
453
|
+
const res = await fetchImpl(outputRecordUrl, {
|
|
454
|
+
method: 'PUT',
|
|
455
|
+
headers: {
|
|
456
|
+
authorization: `Bearer ${token}`,
|
|
457
|
+
'content-type': 'application/json; charset=utf-8',
|
|
458
|
+
},
|
|
459
|
+
body: JSON.stringify(outputRecord),
|
|
460
|
+
signal: controller.signal,
|
|
461
|
+
});
|
|
462
|
+
if (res.status < 200 || res.status >= 300) {
|
|
463
|
+
throw new Error(`Apify KVS saveOutput HTTP ${res.status}`);
|
|
464
|
+
}
|
|
465
|
+
} finally {
|
|
466
|
+
clearTimeout(timer);
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
if (datasetId) {
|
|
472
|
+
const datasetItemsUrl = `${apiBase}/v2/datasets/${encodeURIComponent(datasetId)}/items`;
|
|
473
|
+
hooks.listDeliveredIds = async () => {
|
|
474
|
+
const controller = new AbortController();
|
|
475
|
+
const timer = setTimeout(() => controller.abort(), 1500);
|
|
476
|
+
try {
|
|
477
|
+
const res = await fetchImpl(`${datasetItemsUrl}?fields=creativeId&clean=1`, {
|
|
478
|
+
method: 'GET',
|
|
479
|
+
headers: {
|
|
480
|
+
authorization: `Bearer ${token}`,
|
|
481
|
+
accept: 'application/json',
|
|
482
|
+
},
|
|
483
|
+
signal: controller.signal,
|
|
484
|
+
});
|
|
485
|
+
if (res.status >= 200 && res.status < 300) {
|
|
486
|
+
const text = await res.text();
|
|
487
|
+
const parsed = text && text.trim() ? JSON.parse(text) : [];
|
|
488
|
+
const arr = Array.isArray(parsed)
|
|
489
|
+
? parsed
|
|
490
|
+
: Array.isArray(parsed?.data?.items)
|
|
491
|
+
? parsed.data.items
|
|
492
|
+
: [];
|
|
493
|
+
return arr
|
|
494
|
+
.map((row) => (typeof row?.creativeId === 'string' ? row.creativeId : null))
|
|
495
|
+
.filter(Boolean);
|
|
496
|
+
}
|
|
497
|
+
throw new Error(`Apify Dataset listDeliveredIds failed with HTTP ${res.status}`);
|
|
498
|
+
} finally {
|
|
499
|
+
clearTimeout(timer);
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
hooks.pushData = async (items) => {
|
|
504
|
+
const controller = new AbortController();
|
|
505
|
+
const timer = setTimeout(() => controller.abort(), 1500);
|
|
506
|
+
try {
|
|
507
|
+
const res = await fetchImpl(datasetItemsUrl, {
|
|
508
|
+
method: 'POST',
|
|
509
|
+
headers: {
|
|
510
|
+
authorization: `Bearer ${token}`,
|
|
511
|
+
'content-type': 'application/json; charset=utf-8',
|
|
512
|
+
},
|
|
513
|
+
body: JSON.stringify(items),
|
|
514
|
+
signal: controller.signal,
|
|
515
|
+
});
|
|
516
|
+
if (res.status < 200 || res.status >= 300) {
|
|
517
|
+
throw new Error(`Apify Dataset pushData HTTP ${res.status}`);
|
|
518
|
+
}
|
|
519
|
+
} finally {
|
|
520
|
+
clearTimeout(timer);
|
|
521
|
+
}
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (runId) {
|
|
526
|
+
const chargeUrl = `${apiBase}/v2/actor-runs/${encodeURIComponent(runId)}/charge`;
|
|
527
|
+
const allowUnbilledAtHome = env?.ADPULSE_ALLOW_UNBILLED_AT_HOME === '1';
|
|
528
|
+
let unbilledOwnerTestLatched = false;
|
|
529
|
+
let latchedPlatformEventName = null;
|
|
530
|
+
const resolvePlatformEventAndPrice = (rawEventName) => {
|
|
531
|
+
const candidate = String(rawEventName || ACTOR_PRICING_CONTRACT.eventName);
|
|
532
|
+
const effectiveName =
|
|
533
|
+
latchedPlatformEventName ||
|
|
534
|
+
(() => {
|
|
535
|
+
if (pricingInfoChargeEvents) {
|
|
536
|
+
if (Object.prototype.hasOwnProperty.call(pricingInfoChargeEvents, candidate)) {
|
|
537
|
+
return candidate;
|
|
538
|
+
}
|
|
539
|
+
if (
|
|
540
|
+
candidate === 'ad-creative-scraped' ||
|
|
541
|
+
candidate === 'ad_transparency_creative_retrieved'
|
|
542
|
+
) {
|
|
543
|
+
if (
|
|
544
|
+
Object.prototype.hasOwnProperty.call(
|
|
545
|
+
pricingInfoChargeEvents,
|
|
546
|
+
'ad_transparency_creative_retrieved'
|
|
547
|
+
)
|
|
548
|
+
) {
|
|
549
|
+
return 'ad_transparency_creative_retrieved';
|
|
550
|
+
}
|
|
551
|
+
if (
|
|
552
|
+
Object.prototype.hasOwnProperty.call(
|
|
553
|
+
pricingInfoChargeEvents,
|
|
554
|
+
'ad-creative-scraped'
|
|
555
|
+
)
|
|
556
|
+
) {
|
|
557
|
+
return 'ad-creative-scraped';
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
return candidate;
|
|
562
|
+
})();
|
|
563
|
+
const rawEventPrice = Number(pricingInfoChargeEvents?.[effectiveName]?.eventPriceUsd);
|
|
564
|
+
const unitPriceUsdc =
|
|
565
|
+
Number.isFinite(rawEventPrice) && rawEventPrice > 0
|
|
566
|
+
? rawEventPrice
|
|
567
|
+
: ACTOR_PRICING_CONTRACT.priceUsdc;
|
|
568
|
+
return { eventName: effectiveName, unitPriceUsdc };
|
|
569
|
+
};
|
|
570
|
+
hooks.unitPriceUsdc = resolvePlatformEventAndPrice(
|
|
571
|
+
ACTOR_PRICING_CONTRACT.eventName
|
|
572
|
+
).unitPriceUsdc;
|
|
573
|
+
hooks.chargeEvent = async (receipt) => {
|
|
574
|
+
const requestedCount = Math.max(0, Number(receipt?.count) || 0);
|
|
575
|
+
const { eventName: primaryEventName, unitPriceUsdc: resolvedUnitPriceUsdc } =
|
|
576
|
+
resolvePlatformEventAndPrice(receipt?.eventName);
|
|
577
|
+
hooks.unitPriceUsdc = resolvedUnitPriceUsdc;
|
|
578
|
+
if (unbilledOwnerTestLatched) {
|
|
579
|
+
return {
|
|
580
|
+
chargedCount: 0,
|
|
581
|
+
eventChargeLimitReached: false,
|
|
582
|
+
unbilledOwnerTestBypass: true,
|
|
583
|
+
chargedCreativeIds: [],
|
|
584
|
+
eventName: primaryEventName,
|
|
585
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
if (adapterChargeLimitLatched) {
|
|
589
|
+
return {
|
|
590
|
+
chargedCount: 0,
|
|
591
|
+
eventChargeLimitReached: true,
|
|
592
|
+
chargedCreativeIds: [],
|
|
593
|
+
eventName: primaryEventName,
|
|
594
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
const unitPriceMicros = Math.round(resolvedUnitPriceUsdc * 1_000_000);
|
|
598
|
+
if (!Number.isFinite(unitPriceMicros) || unitPriceMicros < 1) {
|
|
599
|
+
throw new Error(
|
|
600
|
+
`Invalid unitPriceUsdc (${String(resolvedUnitPriceUsdc)}): unitPriceMicros must be >= 1`
|
|
601
|
+
);
|
|
602
|
+
}
|
|
603
|
+
const maxPayableEvents = hasMaxChargeUsd
|
|
604
|
+
? Math.floor(Math.max(0, maxBudgetMicros - startFeeMicros) / unitPriceMicros)
|
|
605
|
+
: Infinity;
|
|
606
|
+
const remainingAllowance = Number.isFinite(maxPayableEvents)
|
|
607
|
+
? Math.max(0, maxPayableEvents - cumulativeChargedInAdapter)
|
|
608
|
+
: requestedCount;
|
|
609
|
+
const countToCharge = Math.min(requestedCount, remainingAllowance);
|
|
610
|
+
const limitReached =
|
|
611
|
+
Number.isFinite(maxPayableEvents) &&
|
|
612
|
+
cumulativeChargedInAdapter + countToCharge >= maxPayableEvents &&
|
|
613
|
+
countToCharge < requestedCount;
|
|
614
|
+
|
|
615
|
+
if (countToCharge === 0) {
|
|
616
|
+
return {
|
|
617
|
+
chargedCount: 0,
|
|
618
|
+
eventChargeLimitReached: Number.isFinite(maxPayableEvents),
|
|
619
|
+
chargedCreativeIds: [],
|
|
620
|
+
eventName: primaryEventName,
|
|
621
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
const chargeIdempotencyVerified =
|
|
626
|
+
env?.APIFY_CHARGE_IDEMPOTENCY_SUPPORTED === '1' ||
|
|
627
|
+
env?.APIFY_CHARGE_IDEMPOTENCY_SUPPORTED === 'true';
|
|
628
|
+
const maxChargeAttempts = chargeIdempotencyVerified ? 2 : 1;
|
|
629
|
+
const perAttemptTimeoutMs = 900;
|
|
630
|
+
const canStartChargePost = () =>
|
|
631
|
+
!Number.isFinite(receipt?.deadlineAtMs) ||
|
|
632
|
+
Date.now() + perAttemptTimeoutMs + 300 < receipt.deadlineAtMs;
|
|
633
|
+
|
|
634
|
+
const postChargeRequestWithIdempotentRetry = async (
|
|
635
|
+
eventNameToPost,
|
|
636
|
+
idempotencyKeyToPost
|
|
637
|
+
) => {
|
|
638
|
+
let lastPostErr = null;
|
|
639
|
+
for (let attempt = 1; attempt <= maxChargeAttempts; attempt += 1) {
|
|
640
|
+
if (attempt > 1 && !canStartChargePost()) {
|
|
641
|
+
break;
|
|
642
|
+
}
|
|
643
|
+
const controller = new AbortController();
|
|
644
|
+
const timer = setTimeout(() => controller.abort(), perAttemptTimeoutMs);
|
|
645
|
+
try {
|
|
646
|
+
const res = await fetchImpl(chargeUrl, {
|
|
647
|
+
method: 'POST',
|
|
648
|
+
headers: {
|
|
649
|
+
authorization: `Bearer ${token}`,
|
|
650
|
+
'content-type': 'application/json; charset=utf-8',
|
|
651
|
+
'idempotency-key': idempotencyKeyToPost,
|
|
652
|
+
},
|
|
653
|
+
body: JSON.stringify({
|
|
654
|
+
eventName: eventNameToPost,
|
|
655
|
+
count: countToCharge,
|
|
656
|
+
}),
|
|
657
|
+
signal: controller.signal,
|
|
658
|
+
});
|
|
659
|
+
if (
|
|
660
|
+
attempt < maxChargeAttempts &&
|
|
661
|
+
(res.status === 429 || res.status >= 500) &&
|
|
662
|
+
canStartChargePost()
|
|
663
|
+
) {
|
|
664
|
+
try {
|
|
665
|
+
await res.body?.cancel?.();
|
|
666
|
+
} catch {
|
|
667
|
+
// Ignore body cancel failure on retry
|
|
668
|
+
}
|
|
669
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
const bodyText =
|
|
673
|
+
typeof res?.text === 'function'
|
|
674
|
+
? await Promise.race([
|
|
675
|
+
res.text(),
|
|
676
|
+
new Promise((_, reject) => {
|
|
677
|
+
if (controller.signal.aborted) {
|
|
678
|
+
reject(new Error('AbortError: charge response body timed out'));
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
controller.signal.addEventListener(
|
|
682
|
+
'abort',
|
|
683
|
+
() => reject(new Error('AbortError: charge response body timed out')),
|
|
684
|
+
{ once: true }
|
|
685
|
+
);
|
|
686
|
+
}),
|
|
687
|
+
])
|
|
688
|
+
: '';
|
|
689
|
+
return { status: res.status, bodyText };
|
|
690
|
+
} catch (err) {
|
|
691
|
+
lastPostErr = err;
|
|
692
|
+
if (attempt < maxChargeAttempts && canStartChargePost()) {
|
|
693
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
throw err;
|
|
697
|
+
} finally {
|
|
698
|
+
clearTimeout(timer);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
throw lastPostErr || new Error('Charge POST exceeded run deadline');
|
|
702
|
+
};
|
|
703
|
+
|
|
704
|
+
try {
|
|
705
|
+
let effectiveEventName = primaryEventName;
|
|
706
|
+
let effectiveIdempotencyKey = receipt.idempotencyKey;
|
|
707
|
+
let res = await postChargeRequestWithIdempotentRetry(
|
|
708
|
+
primaryEventName,
|
|
709
|
+
receipt.idempotencyKey
|
|
710
|
+
);
|
|
711
|
+
if (
|
|
712
|
+
res.status === 404 &&
|
|
713
|
+
!pricingInfoChargeEvents &&
|
|
714
|
+
!latchedPlatformEventName &&
|
|
715
|
+
(primaryEventName === 'ad-creative-scraped' ||
|
|
716
|
+
primaryEventName === 'ad_transparency_creative_retrieved')
|
|
717
|
+
) {
|
|
718
|
+
const aliasEventName =
|
|
719
|
+
primaryEventName === 'ad-creative-scraped'
|
|
720
|
+
? 'ad_transparency_creative_retrieved'
|
|
721
|
+
: 'ad-creative-scraped';
|
|
722
|
+
const aliasIdempotencyKey = `${receipt.idempotencyKey}-${aliasEventName}`;
|
|
723
|
+
const fallbackRes = await postChargeRequestWithIdempotentRetry(
|
|
724
|
+
aliasEventName,
|
|
725
|
+
aliasIdempotencyKey
|
|
726
|
+
);
|
|
727
|
+
if (fallbackRes.status !== 404) {
|
|
728
|
+
res = fallbackRes;
|
|
729
|
+
effectiveEventName = aliasEventName;
|
|
730
|
+
effectiveIdempotencyKey = aliasIdempotencyKey;
|
|
731
|
+
if (
|
|
732
|
+
(fallbackRes.status >= 200 && fallbackRes.status < 300) ||
|
|
733
|
+
fallbackRes.status === 402
|
|
734
|
+
) {
|
|
735
|
+
latchedPlatformEventName = aliasEventName;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
if (res.status === 402) {
|
|
740
|
+
// Platform enforced spending ceiling (HTTP 402 Payment Required): latch so no further charge POSTs occur
|
|
741
|
+
adapterChargeLimitLatched = true;
|
|
742
|
+
return {
|
|
743
|
+
chargedCount: 0,
|
|
744
|
+
eventChargeLimitReached: true,
|
|
745
|
+
chargedCreativeIds: [],
|
|
746
|
+
eventName: effectiveEventName,
|
|
747
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
748
|
+
idempotencyKey: effectiveIdempotencyKey,
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
if (allowUnbilledAtHome && (res.status === 405 || res.status === 403)) {
|
|
752
|
+
let isUnmonetizedNotAllowed = res.status === 405;
|
|
753
|
+
if (res.status === 403) {
|
|
754
|
+
try {
|
|
755
|
+
const errText = res.bodyText || '';
|
|
756
|
+
const errJson = errText && errText.trim() ? JSON.parse(errText) : null;
|
|
757
|
+
isUnmonetizedNotAllowed = errJson?.error?.type === 'operation-not-allowed';
|
|
758
|
+
} catch {
|
|
759
|
+
isUnmonetizedNotAllowed = false;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
if (isUnmonetizedNotAllowed) {
|
|
763
|
+
unbilledOwnerTestLatched = true;
|
|
764
|
+
return {
|
|
765
|
+
chargedCount: 0,
|
|
766
|
+
eventChargeLimitReached: false,
|
|
767
|
+
unbilledOwnerTestBypass: true,
|
|
768
|
+
chargedCreativeIds: [],
|
|
769
|
+
eventName: effectiveEventName,
|
|
770
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
771
|
+
idempotencyKey: effectiveIdempotencyKey,
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
if (res.status < 200 || res.status >= 300) {
|
|
776
|
+
let errSuffix = '';
|
|
777
|
+
try {
|
|
778
|
+
const errText = res.bodyText || '';
|
|
779
|
+
if (errText && errText.trim()) {
|
|
780
|
+
const errJson = JSON.parse(errText);
|
|
781
|
+
const errType = errJson?.error?.type;
|
|
782
|
+
if (typeof errType === 'string' && /^[A-Za-z0-9_.:-]{1,64}$/.test(errType)) {
|
|
783
|
+
errSuffix = ` (${errType})`;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
} catch {
|
|
787
|
+
errSuffix = '';
|
|
788
|
+
}
|
|
789
|
+
throw new Error(`Apify Pay-per-Event charge HTTP ${res.status}${errSuffix}`);
|
|
790
|
+
}
|
|
791
|
+
const rawText = res.bodyText || '';
|
|
792
|
+
const parsed = rawText && rawText.trim() ? JSON.parse(rawText) : {};
|
|
793
|
+
const dataObj = parsed?.data && typeof parsed.data === 'object' ? parsed.data : parsed;
|
|
794
|
+
// Apify REST `POST /v2/actor-runs/{runId}/charge` returns HTTP 201 Created with `{}` on success.
|
|
795
|
+
// Only fallback to `countToCharge` when `chargedCount` is genuinely omitted; if present but invalid, throw fail-closed.
|
|
796
|
+
const hasExplicitChargedCount =
|
|
797
|
+
Boolean(dataObj) &&
|
|
798
|
+
typeof dataObj === 'object' &&
|
|
799
|
+
Object.prototype.hasOwnProperty.call(dataObj, 'chargedCount') &&
|
|
800
|
+
dataObj.chargedCount !== undefined;
|
|
801
|
+
if (
|
|
802
|
+
hasExplicitChargedCount &&
|
|
803
|
+
(!Number.isInteger(dataObj.chargedCount) ||
|
|
804
|
+
dataObj.chargedCount < 0 ||
|
|
805
|
+
dataObj.chargedCount > countToCharge)
|
|
806
|
+
) {
|
|
807
|
+
throw new Error(
|
|
808
|
+
`Apify Pay-per-Event charge response contained invalid chargedCount: ${String(dataObj.chargedCount)}`
|
|
809
|
+
);
|
|
810
|
+
}
|
|
811
|
+
const resolvedChargedCount = hasExplicitChargedCount
|
|
812
|
+
? dataObj.chargedCount
|
|
813
|
+
: countToCharge;
|
|
814
|
+
cumulativeChargedInAdapter += resolvedChargedCount;
|
|
815
|
+
const resolvedLimitReached =
|
|
816
|
+
Boolean(dataObj?.eventChargeLimitReached) ||
|
|
817
|
+
limitReached ||
|
|
818
|
+
(Number.isFinite(maxPayableEvents) && cumulativeChargedInAdapter >= maxPayableEvents);
|
|
819
|
+
let subsetIds;
|
|
820
|
+
if (Array.isArray(dataObj?.chargedCreativeIds)) {
|
|
821
|
+
const reqIds = Array.isArray(receipt?.creativeIds) ? receipt.creativeIds : null;
|
|
822
|
+
const uniqueValidIds = new Set(
|
|
823
|
+
dataObj.chargedCreativeIds.filter(
|
|
824
|
+
(id) => typeof id === 'string' && (!reqIds || reqIds.includes(id))
|
|
825
|
+
)
|
|
826
|
+
);
|
|
827
|
+
if (
|
|
828
|
+
dataObj.chargedCreativeIds.length !== resolvedChargedCount ||
|
|
829
|
+
uniqueValidIds.size !== resolvedChargedCount
|
|
830
|
+
) {
|
|
831
|
+
throw new Error(
|
|
832
|
+
'Apify Pay-per-Event charge response contained invalid chargedCreativeIds'
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
subsetIds = Array.from(uniqueValidIds);
|
|
836
|
+
} else if (Array.isArray(receipt?.creativeIds)) {
|
|
837
|
+
subsetIds = receipt.creativeIds.slice(0, resolvedChargedCount);
|
|
838
|
+
}
|
|
839
|
+
return {
|
|
840
|
+
chargedCount: resolvedChargedCount,
|
|
841
|
+
eventChargeLimitReached: resolvedLimitReached,
|
|
842
|
+
eventName: effectiveEventName,
|
|
843
|
+
unitPriceUsdc: resolvedUnitPriceUsdc,
|
|
844
|
+
idempotencyKey: effectiveIdempotencyKey,
|
|
845
|
+
...(subsetIds ? { chargedCreativeIds: subsetIds } : {}),
|
|
846
|
+
};
|
|
847
|
+
} finally {
|
|
848
|
+
// Per-attempt timeout timers are cleared inside postChargeRequestWithIdempotentRetry
|
|
849
|
+
}
|
|
850
|
+
};
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
return hooks;
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
// Local Apify storage directory mode (`APIFY_LOCAL_STORAGE_DIR`)
|
|
857
|
+
const storeName = kvsId || 'default';
|
|
858
|
+
const dsName = datasetId || 'default';
|
|
859
|
+
const kvsDir = path.join(localStorageDir, 'key_value_stores', storeName);
|
|
860
|
+
const dsDir = path.join(localStorageDir, 'datasets', dsName);
|
|
861
|
+
const checkpointPath = path.join(kvsDir, `${ACTOR_PRICING_CONTRACT.checkpointKey}.json`);
|
|
862
|
+
const outputPath = path.join(kvsDir, 'OUTPUT.json');
|
|
863
|
+
|
|
864
|
+
hooks.loadCheckpoint = async () => {
|
|
865
|
+
if (!fs.existsSync(checkpointPath)) return null;
|
|
866
|
+
return JSON.parse(fs.readFileSync(checkpointPath, 'utf8'));
|
|
867
|
+
};
|
|
868
|
+
|
|
869
|
+
hooks.saveCheckpoint = async (snapshot) => {
|
|
870
|
+
fs.mkdirSync(kvsDir, { recursive: true });
|
|
871
|
+
fs.writeFileSync(checkpointPath, JSON.stringify(snapshot, null, 2), 'utf8');
|
|
872
|
+
};
|
|
873
|
+
|
|
874
|
+
hooks.saveOutput = async (outputRecord) => {
|
|
875
|
+
fs.mkdirSync(kvsDir, { recursive: true });
|
|
876
|
+
fs.writeFileSync(outputPath, JSON.stringify(outputRecord, null, 2), 'utf8');
|
|
877
|
+
};
|
|
878
|
+
|
|
879
|
+
hooks.listDeliveredIds = async () => {
|
|
880
|
+
if (!fs.existsSync(dsDir)) return [];
|
|
881
|
+
const existingFiles = fs.readdirSync(dsDir).filter((f) => f.endsWith('.json'));
|
|
882
|
+
const ids = [];
|
|
883
|
+
for (const file of existingFiles) {
|
|
884
|
+
try {
|
|
885
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(dsDir, file), 'utf8'));
|
|
886
|
+
if (typeof parsed?.creativeId === 'string') {
|
|
887
|
+
ids.push(parsed.creativeId);
|
|
888
|
+
}
|
|
889
|
+
} catch {
|
|
890
|
+
// Ignore unreadable partial files
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
return ids;
|
|
894
|
+
};
|
|
895
|
+
|
|
896
|
+
hooks.pushData = async (items) => {
|
|
897
|
+
fs.mkdirSync(dsDir, { recursive: true });
|
|
898
|
+
const existingSeqs = fs
|
|
899
|
+
.readdirSync(dsDir)
|
|
900
|
+
.map((f) => {
|
|
901
|
+
const m = /^(\d+)\.json$/.exec(f);
|
|
902
|
+
return m ? Number(m[1]) : 0;
|
|
903
|
+
})
|
|
904
|
+
.filter((n) => Number.isInteger(n) && n > 0);
|
|
905
|
+
let nextSeq = existingSeqs.length > 0 ? Math.max(...existingSeqs) + 1 : 1;
|
|
906
|
+
const list = Array.isArray(items) ? items : [items];
|
|
907
|
+
for (const item of list) {
|
|
908
|
+
const fileName = `${String(nextSeq++).padStart(9, '0')}.json`;
|
|
909
|
+
fs.writeFileSync(path.join(dsDir, fileName), JSON.stringify(item, null, 2), 'utf8');
|
|
910
|
+
}
|
|
911
|
+
};
|
|
912
|
+
|
|
913
|
+
return hooks;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/**
|
|
917
|
+
* Normalizes a persisted KVS billing checkpoint object.
|
|
918
|
+
* If the checkpoint belongs to a different `runId`, discards stale state and starts fresh.
|
|
919
|
+
*
|
|
920
|
+
* @param {unknown} rawState
|
|
921
|
+
* @param {string} runId
|
|
922
|
+
* @returns {object}
|
|
923
|
+
*/
|
|
924
|
+
function normalizeBillingCheckpointState(rawState, runId) {
|
|
925
|
+
const s = rawState && typeof rawState === 'object' && !Array.isArray(rawState) ? rawState : {};
|
|
926
|
+
if (s.runId && String(s.runId).trim() !== String(runId).trim()) {
|
|
927
|
+
return {
|
|
928
|
+
runId: String(runId || 'default_run'),
|
|
929
|
+
chargedIds: [],
|
|
930
|
+
deliveredIds: [],
|
|
931
|
+
inFlightIds: [],
|
|
932
|
+
inFlightIdempotencyKey: null,
|
|
933
|
+
unknownOutcomeIds: [],
|
|
934
|
+
stagedItemsById: {},
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
const toUniqueIdArray = (arr) =>
|
|
939
|
+
Array.isArray(arr)
|
|
940
|
+
? Array.from(new Set(arr.map((x) => String(x || '').trim()).filter((x) => CREATIVE_ID_RE.test(x))))
|
|
941
|
+
: [];
|
|
942
|
+
|
|
943
|
+
const stagedItemsById = {};
|
|
944
|
+
if (s.stagedItemsById && typeof s.stagedItemsById === 'object' && !Array.isArray(s.stagedItemsById)) {
|
|
945
|
+
for (const [k, v] of Object.entries(s.stagedItemsById)) {
|
|
946
|
+
if (CREATIVE_ID_RE.test(k) && v && typeof v === 'object') {
|
|
947
|
+
stagedItemsById[k] = v;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
return {
|
|
953
|
+
runId: String(s.runId || runId || 'default_run'),
|
|
954
|
+
chargedIds: toUniqueIdArray(s.chargedIds),
|
|
955
|
+
deliveredIds: toUniqueIdArray(s.deliveredIds),
|
|
956
|
+
inFlightIds: toUniqueIdArray(s.inFlightIds),
|
|
957
|
+
inFlightIdempotencyKey:
|
|
958
|
+
typeof s.inFlightIdempotencyKey === 'string' && s.inFlightIdempotencyKey.trim()
|
|
959
|
+
? s.inFlightIdempotencyKey.trim()
|
|
960
|
+
: null,
|
|
961
|
+
unknownOutcomeIds: toUniqueIdArray(s.unknownOutcomeIds),
|
|
962
|
+
stagedItemsById,
|
|
963
|
+
};
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Executes the end-to-end Google Ads Transparency Actor workflow with crash-safe billing
|
|
968
|
+
* checkpointing, strict `ChargeResult` validation, deterministic partial-limit prefix delivery,
|
|
969
|
+
* mid-stream cursor rewind, and fail-closed zero-charge protection.
|
|
970
|
+
*
|
|
971
|
+
* @param {object} input
|
|
972
|
+
* @param {object} [hooks]
|
|
973
|
+
* @param {Function} [hooks.pushData]
|
|
974
|
+
* @param {Function} [hooks.chargeEvent]
|
|
975
|
+
* @param {Function} [hooks.loadCheckpoint]
|
|
976
|
+
* @param {Function} [hooks.saveCheckpoint]
|
|
977
|
+
* @param {Function} [hooks.fetchImpl]
|
|
978
|
+
* @param {Function} [hooks.nowMs]
|
|
979
|
+
* @param {string} [hooks.runId]
|
|
980
|
+
* @returns {Promise<object>}
|
|
981
|
+
*/
|
|
982
|
+
export async function runGoogleAdsTransparencyActor(input = {}, hooks = {}) {
|
|
983
|
+
const nowMs = typeof hooks.nowMs === 'function' ? hooks.nowMs : () => Date.now();
|
|
984
|
+
const startedAtMs = nowMs();
|
|
985
|
+
// Authoritative runId precedence: hooks.runId -> ACTOR_RUN_ID -> APIFY_ACTOR_RUN_ID -> input.runId -> 'default_run'
|
|
986
|
+
const runId = String(
|
|
987
|
+
hooks.runId ||
|
|
988
|
+
process.env.ACTOR_RUN_ID ||
|
|
989
|
+
process.env.APIFY_ACTOR_RUN_ID ||
|
|
990
|
+
input?.runId ||
|
|
991
|
+
'default_run'
|
|
992
|
+
);
|
|
993
|
+
const hasChargeHook = typeof hooks.chargeEvent === 'function';
|
|
994
|
+
let billingMode =
|
|
995
|
+
hasChargeHook || hooks.isAtHome === true ? 'APIFY_PAY_PER_EVENT' : 'LOCAL_UNBILLED';
|
|
996
|
+
|
|
997
|
+
const maxRunDurationMs = Math.min(
|
|
998
|
+
DEFAULT_RUN_DEADLINE_MS,
|
|
999
|
+
Math.max(1000, Number(input?.maxRunDurationMs) || DEFAULT_RUN_DEADLINE_MS)
|
|
1000
|
+
);
|
|
1001
|
+
const deadlineAtMs = startedAtMs + maxRunDurationMs;
|
|
1002
|
+
const billingReserveMs =
|
|
1003
|
+
maxRunDurationMs >= 6000 ? 2500 : Math.min(1000, Math.floor(maxRunDurationMs * 0.25));
|
|
1004
|
+
const rpcDeadlineAtMs = hasChargeHook
|
|
1005
|
+
? Math.max(startedAtMs + 500, deadlineAtMs - billingReserveMs)
|
|
1006
|
+
: deadlineAtMs;
|
|
1007
|
+
const step4ReserveMs = hasChargeHook
|
|
1008
|
+
? Math.min(2100, Math.floor(billingReserveMs * 0.85))
|
|
1009
|
+
: 0;
|
|
1010
|
+
|
|
1011
|
+
const finalizeActorReturn = async (finalResult) => {
|
|
1012
|
+
if (typeof hooks.saveOutput === 'function' && nowMs() < deadlineAtMs) {
|
|
1013
|
+
try {
|
|
1014
|
+
await hooks.saveOutput({
|
|
1015
|
+
ok: finalResult.ok,
|
|
1016
|
+
status: finalResult.status,
|
|
1017
|
+
reason: finalResult.reason || null,
|
|
1018
|
+
billingMode: finalResult.billingMode,
|
|
1019
|
+
extractedCount: finalResult.extractedCount,
|
|
1020
|
+
deliveredCount: finalResult.deliveredCount,
|
|
1021
|
+
billableCount: finalResult.billableCount,
|
|
1022
|
+
chargedEventCount: finalResult.chargedEventCount,
|
|
1023
|
+
pagination: finalResult.pagination,
|
|
1024
|
+
...(Array.isArray(finalResult.warnings) && finalResult.warnings.length > 0
|
|
1025
|
+
? { warnings: finalResult.warnings }
|
|
1026
|
+
: {}),
|
|
1027
|
+
queryMeta: finalResult.queryMeta,
|
|
1028
|
+
selectedEntities: finalResult.selectedEntities,
|
|
1029
|
+
telemetry: finalResult.telemetry,
|
|
1030
|
+
});
|
|
1031
|
+
} catch {
|
|
1032
|
+
// Non-fatal KVS summary write
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
return finalResult;
|
|
1036
|
+
};
|
|
1037
|
+
|
|
1038
|
+
const pushDataWithRetry = async (itemsToPush) => {
|
|
1039
|
+
if (
|
|
1040
|
+
typeof hooks.pushData !== 'function' ||
|
|
1041
|
+
!Array.isArray(itemsToPush) ||
|
|
1042
|
+
itemsToPush.length === 0
|
|
1043
|
+
) {
|
|
1044
|
+
return { ok: true };
|
|
1045
|
+
}
|
|
1046
|
+
let pendingItems = [...itemsToPush];
|
|
1047
|
+
let lastErr = null;
|
|
1048
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
1049
|
+
try {
|
|
1050
|
+
await hooks.pushData(pendingItems);
|
|
1051
|
+
return { ok: true };
|
|
1052
|
+
} catch (err) {
|
|
1053
|
+
lastErr = err;
|
|
1054
|
+
if (attempt < 2) {
|
|
1055
|
+
if (nowMs() + 1500 >= deadlineAtMs || typeof hooks.listDeliveredIds !== 'function') {
|
|
1056
|
+
return { ok: false, error: lastErr };
|
|
1057
|
+
}
|
|
1058
|
+
try {
|
|
1059
|
+
const committedIds = new Set(await hooks.listDeliveredIds());
|
|
1060
|
+
pendingItems = pendingItems.filter((it) => !committedIds.has(it.creativeId));
|
|
1061
|
+
if (pendingItems.length === 0) {
|
|
1062
|
+
return { ok: true };
|
|
1063
|
+
}
|
|
1064
|
+
} catch (reconcileErr) {
|
|
1065
|
+
return { ok: false, error: reconcileErr || lastErr };
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
return { ok: false, error: lastErr };
|
|
1071
|
+
};
|
|
1072
|
+
|
|
1073
|
+
// Step 0.0: Fail-Closed At-Home Cloud Hook Completeness Gate
|
|
1074
|
+
// In Apify Cloud (`hooks.isAtHome === true`), require all 4 platform hooks (`chargeEvent`,
|
|
1075
|
+
// `pushData`, `loadCheckpoint`, `saveCheckpoint`) before any RPC fetch or billing occurs.
|
|
1076
|
+
if (
|
|
1077
|
+
hooks.isAtHome === true &&
|
|
1078
|
+
(typeof hooks.chargeEvent !== 'function' ||
|
|
1079
|
+
typeof hooks.pushData !== 'function' ||
|
|
1080
|
+
typeof hooks.loadCheckpoint !== 'function' ||
|
|
1081
|
+
typeof hooks.saveCheckpoint !== 'function')
|
|
1082
|
+
) {
|
|
1083
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1084
|
+
return finalizeActorReturn({
|
|
1085
|
+
ok: false,
|
|
1086
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1087
|
+
reason:
|
|
1088
|
+
'Incomplete Apify Cloud runtime hooks (missing chargeEvent, pushData, loadCheckpoint, or saveCheckpoint in at-home mode)',
|
|
1089
|
+
billingMode,
|
|
1090
|
+
queryMeta: null,
|
|
1091
|
+
selectedEntities: [],
|
|
1092
|
+
items: [],
|
|
1093
|
+
extractedCount: 0,
|
|
1094
|
+
deliveredCount: 0,
|
|
1095
|
+
billableCount: 0,
|
|
1096
|
+
chargedEventCount: 0,
|
|
1097
|
+
unknownOutcomeIds: [],
|
|
1098
|
+
chargeReceipts: [],
|
|
1099
|
+
pagination: {
|
|
1100
|
+
nextPageToken: input?.pageToken || null,
|
|
1101
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1102
|
+
paginationIncompleteReason: 'CLOUD_HOOKS_INCOMPLETE',
|
|
1103
|
+
},
|
|
1104
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1105
|
+
billableCount: 0,
|
|
1106
|
+
chargedEventCount: 0,
|
|
1107
|
+
durationMs,
|
|
1108
|
+
requestCount: 1,
|
|
1109
|
+
}),
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
// Step 0: Load & Reconcile Crash-Safe Billing Checkpoint (fail-closed if loadCheckpoint throws)
|
|
1114
|
+
let rawLoadedCheckpoint = null;
|
|
1115
|
+
if (typeof hooks.loadCheckpoint === 'function') {
|
|
1116
|
+
try {
|
|
1117
|
+
rawLoadedCheckpoint = await hooks.loadCheckpoint();
|
|
1118
|
+
} catch (cpErr) {
|
|
1119
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1120
|
+
return finalizeActorReturn({
|
|
1121
|
+
ok: false,
|
|
1122
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1123
|
+
reason: `Failed to load billing checkpoint fail-closed: ${cpErr?.message || String(cpErr)}`,
|
|
1124
|
+
billingMode,
|
|
1125
|
+
queryMeta: null,
|
|
1126
|
+
selectedEntities: [],
|
|
1127
|
+
items: [],
|
|
1128
|
+
extractedCount: 0,
|
|
1129
|
+
deliveredCount: 0,
|
|
1130
|
+
billableCount: 0,
|
|
1131
|
+
chargedEventCount: 0,
|
|
1132
|
+
unknownOutcomeIds: [],
|
|
1133
|
+
chargeReceipts: [],
|
|
1134
|
+
pagination: {
|
|
1135
|
+
nextPageToken: input?.pageToken || null,
|
|
1136
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1137
|
+
paginationIncompleteReason: 'CHECKPOINT_LOAD_FAILED',
|
|
1138
|
+
},
|
|
1139
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1140
|
+
billableCount: 0,
|
|
1141
|
+
chargedEventCount: 0,
|
|
1142
|
+
durationMs,
|
|
1143
|
+
requestCount: 1,
|
|
1144
|
+
}),
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
const checkpoint = normalizeBillingCheckpointState(rawLoadedCheckpoint, runId);
|
|
1149
|
+
|
|
1150
|
+
const chargedSet = new Set(checkpoint.chargedIds);
|
|
1151
|
+
const deliveredSet = new Set(checkpoint.deliveredIds);
|
|
1152
|
+
const unknownOutcomeSet = new Set(checkpoint.unknownOutcomeIds);
|
|
1153
|
+
let inFlightSet = new Set(checkpoint.inFlightIds);
|
|
1154
|
+
let inFlightIdempotencyKey = checkpoint.inFlightIdempotencyKey || null;
|
|
1155
|
+
const stagedItemsById = { ...checkpoint.stagedItemsById };
|
|
1156
|
+
|
|
1157
|
+
const persistCheckpoint = async () => {
|
|
1158
|
+
const snapshot = {
|
|
1159
|
+
runId,
|
|
1160
|
+
chargedIds: Array.from(chargedSet),
|
|
1161
|
+
deliveredIds: Array.from(deliveredSet),
|
|
1162
|
+
inFlightIds: Array.from(inFlightSet),
|
|
1163
|
+
inFlightIdempotencyKey: inFlightSet.size > 0 ? inFlightIdempotencyKey : null,
|
|
1164
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1165
|
+
stagedItemsById,
|
|
1166
|
+
};
|
|
1167
|
+
if (typeof hooks.saveCheckpoint === 'function') {
|
|
1168
|
+
await hooks.saveCheckpoint(snapshot);
|
|
1169
|
+
}
|
|
1170
|
+
return snapshot;
|
|
1171
|
+
};
|
|
1172
|
+
|
|
1173
|
+
const reconciledItems = [];
|
|
1174
|
+
let deliveredCount = deliveredSet.size;
|
|
1175
|
+
|
|
1176
|
+
// Step 0a: Reconcile any already-paid items (`chargedIds \ deliveredIds`) without re-charging.
|
|
1177
|
+
// First check `hooks.listDeliveredIds()` if available so items already committed to dataset before a lost ACK
|
|
1178
|
+
// are marked delivered without duplicate dataset POSTs. Fail closed if dataset listing throws.
|
|
1179
|
+
const paidUndeliveredIds = Array.from(chargedSet).filter((id) => !deliveredSet.has(id));
|
|
1180
|
+
if (paidUndeliveredIds.length > 0) {
|
|
1181
|
+
let alreadyInDatasetSet = new Set();
|
|
1182
|
+
if (typeof hooks.listDeliveredIds === 'function') {
|
|
1183
|
+
try {
|
|
1184
|
+
const existingDsIds = await hooks.listDeliveredIds();
|
|
1185
|
+
if (Array.isArray(existingDsIds)) {
|
|
1186
|
+
alreadyInDatasetSet = new Set(existingDsIds);
|
|
1187
|
+
}
|
|
1188
|
+
} catch (dsListErr) {
|
|
1189
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1190
|
+
return finalizeActorReturn({
|
|
1191
|
+
ok: false,
|
|
1192
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1193
|
+
reason: `Failed to verify delivered dataset IDs during reconciliation: ${dsListErr?.message || String(dsListErr)}`,
|
|
1194
|
+
billingMode,
|
|
1195
|
+
queryMeta: null,
|
|
1196
|
+
selectedEntities: [],
|
|
1197
|
+
items: [],
|
|
1198
|
+
extractedCount: 0,
|
|
1199
|
+
deliveredCount,
|
|
1200
|
+
billableCount: 0,
|
|
1201
|
+
chargedEventCount: chargedSet.size,
|
|
1202
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1203
|
+
chargeReceipts: [],
|
|
1204
|
+
pagination: {
|
|
1205
|
+
nextPageToken: input?.pageToken || null,
|
|
1206
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1207
|
+
paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
|
|
1208
|
+
},
|
|
1209
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1210
|
+
billableCount: 0,
|
|
1211
|
+
chargedEventCount: chargedSet.size,
|
|
1212
|
+
durationMs,
|
|
1213
|
+
requestCount: 1,
|
|
1214
|
+
}),
|
|
1215
|
+
});
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
const deliverablePairs = [];
|
|
1220
|
+
for (const id of paidUndeliveredIds) {
|
|
1221
|
+
const staged = stagedItemsById[id];
|
|
1222
|
+
if (alreadyInDatasetSet.has(id)) {
|
|
1223
|
+
deliveredSet.add(id);
|
|
1224
|
+
if (staged && typeof staged === 'object' && staged.creativeId === id) {
|
|
1225
|
+
reconciledItems.push(staged);
|
|
1226
|
+
}
|
|
1227
|
+
delete stagedItemsById[id];
|
|
1228
|
+
} else if (staged && typeof staged === 'object' && staged.creativeId === id) {
|
|
1229
|
+
deliverablePairs.push({ id, item: staged });
|
|
1230
|
+
} else {
|
|
1231
|
+
unknownOutcomeSet.add(id);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
if (deliverablePairs.length > 0) {
|
|
1236
|
+
const itemsToDeliver = deliverablePairs.map((p) => p.item);
|
|
1237
|
+
const pushRes = await pushDataWithRetry(itemsToDeliver);
|
|
1238
|
+
if (!pushRes.ok) {
|
|
1239
|
+
await persistCheckpoint();
|
|
1240
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1241
|
+
return finalizeActorReturn({
|
|
1242
|
+
ok: false,
|
|
1243
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1244
|
+
reason: `Dataset pushData failed during reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
|
|
1245
|
+
billingMode,
|
|
1246
|
+
queryMeta: null,
|
|
1247
|
+
selectedEntities: [],
|
|
1248
|
+
items: reconciledItems,
|
|
1249
|
+
extractedCount: 0,
|
|
1250
|
+
deliveredCount,
|
|
1251
|
+
billableCount: reconciledItems.length,
|
|
1252
|
+
chargedEventCount: chargedSet.size,
|
|
1253
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1254
|
+
chargeReceipts: [],
|
|
1255
|
+
pagination: {
|
|
1256
|
+
nextPageToken: input?.pageToken || null,
|
|
1257
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1258
|
+
paginationIncompleteReason: 'DATASET_PUSH_FAILED',
|
|
1259
|
+
},
|
|
1260
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1261
|
+
billableCount: reconciledItems.length,
|
|
1262
|
+
chargedEventCount: chargedSet.size,
|
|
1263
|
+
durationMs,
|
|
1264
|
+
requestCount: 1,
|
|
1265
|
+
}),
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
for (const { id, item } of deliverablePairs) {
|
|
1269
|
+
deliveredSet.add(id);
|
|
1270
|
+
delete stagedItemsById[id];
|
|
1271
|
+
reconciledItems.push(item);
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
deliveredCount = deliveredSet.size;
|
|
1275
|
+
await persistCheckpoint();
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
// Step 0b: Reconcile leftover `inFlightIds` via stored `inFlightIdempotencyKey` when available; otherwise transition to `unknownOutcomeIds`
|
|
1279
|
+
if (inFlightSet.size > 0) {
|
|
1280
|
+
const inFlightArr = Array.from(inFlightSet);
|
|
1281
|
+
const canReplayInFlight =
|
|
1282
|
+
hooks.idempotentChargeRetryEnabled === true &&
|
|
1283
|
+
Boolean(inFlightIdempotencyKey) &&
|
|
1284
|
+
typeof hooks.chargeEvent === 'function' &&
|
|
1285
|
+
inFlightArr.every((id) => stagedItemsById[id] && stagedItemsById[id].creativeId === id);
|
|
1286
|
+
|
|
1287
|
+
let reconciledInFlight = false;
|
|
1288
|
+
if (canReplayInFlight) {
|
|
1289
|
+
try {
|
|
1290
|
+
const replayRes = await hooks.chargeEvent({
|
|
1291
|
+
idempotencyKey: inFlightIdempotencyKey,
|
|
1292
|
+
eventName: ACTOR_PRICING_CONTRACT.eventName,
|
|
1293
|
+
count: inFlightArr.length,
|
|
1294
|
+
unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
|
|
1295
|
+
chunkAmountUsdc: Number(
|
|
1296
|
+
(inFlightArr.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
|
|
1297
|
+
),
|
|
1298
|
+
creativeIds: inFlightArr,
|
|
1299
|
+
deadlineAtMs,
|
|
1300
|
+
});
|
|
1301
|
+
if (replayRes && replayRes.chargedCount === inFlightArr.length) {
|
|
1302
|
+
for (const id of inFlightArr) {
|
|
1303
|
+
chargedSet.add(id);
|
|
1304
|
+
}
|
|
1305
|
+
inFlightSet = new Set();
|
|
1306
|
+
inFlightIdempotencyKey = null;
|
|
1307
|
+
await persistCheckpoint();
|
|
1308
|
+
|
|
1309
|
+
let alreadyInDatasetSet = new Set();
|
|
1310
|
+
if (typeof hooks.listDeliveredIds === 'function') {
|
|
1311
|
+
try {
|
|
1312
|
+
const existingDsIds = await hooks.listDeliveredIds();
|
|
1313
|
+
if (Array.isArray(existingDsIds)) {
|
|
1314
|
+
alreadyInDatasetSet = new Set(existingDsIds);
|
|
1315
|
+
}
|
|
1316
|
+
} catch (dsListErr) {
|
|
1317
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1318
|
+
return finalizeActorReturn({
|
|
1319
|
+
ok: false,
|
|
1320
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1321
|
+
reason: `Failed to verify delivered dataset IDs during in-flight replay reconciliation: ${dsListErr?.message || String(dsListErr)}`,
|
|
1322
|
+
billingMode,
|
|
1323
|
+
queryMeta: null,
|
|
1324
|
+
selectedEntities: [],
|
|
1325
|
+
items: reconciledItems,
|
|
1326
|
+
extractedCount: 0,
|
|
1327
|
+
deliveredCount,
|
|
1328
|
+
billableCount: reconciledItems.length,
|
|
1329
|
+
chargedEventCount: chargedSet.size,
|
|
1330
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1331
|
+
chargeReceipts: [],
|
|
1332
|
+
pagination: {
|
|
1333
|
+
nextPageToken: input?.pageToken || null,
|
|
1334
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1335
|
+
paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
|
|
1336
|
+
},
|
|
1337
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1338
|
+
billableCount: reconciledItems.length,
|
|
1339
|
+
chargedEventCount: chargedSet.size,
|
|
1340
|
+
durationMs,
|
|
1341
|
+
requestCount: 1,
|
|
1342
|
+
}),
|
|
1343
|
+
});
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
const itemsToDeliver = [];
|
|
1348
|
+
for (const id of inFlightArr) {
|
|
1349
|
+
const staged = stagedItemsById[id];
|
|
1350
|
+
if (alreadyInDatasetSet.has(id)) {
|
|
1351
|
+
deliveredSet.add(id);
|
|
1352
|
+
if (staged) reconciledItems.push(staged);
|
|
1353
|
+
delete stagedItemsById[id];
|
|
1354
|
+
} else if (staged) {
|
|
1355
|
+
itemsToDeliver.push(staged);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
if (itemsToDeliver.length > 0) {
|
|
1360
|
+
const pushRes = await pushDataWithRetry(itemsToDeliver);
|
|
1361
|
+
if (!pushRes.ok) {
|
|
1362
|
+
await persistCheckpoint();
|
|
1363
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1364
|
+
return finalizeActorReturn({
|
|
1365
|
+
ok: false,
|
|
1366
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1367
|
+
reason: `Dataset pushData failed during in-flight replay reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
|
|
1368
|
+
billingMode,
|
|
1369
|
+
queryMeta: null,
|
|
1370
|
+
selectedEntities: [],
|
|
1371
|
+
items: reconciledItems,
|
|
1372
|
+
extractedCount: 0,
|
|
1373
|
+
deliveredCount: deliveredSet.size,
|
|
1374
|
+
billableCount: reconciledItems.length,
|
|
1375
|
+
chargedEventCount: chargedSet.size,
|
|
1376
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1377
|
+
chargeReceipts: [],
|
|
1378
|
+
pagination: {
|
|
1379
|
+
nextPageToken: input?.pageToken || null,
|
|
1380
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1381
|
+
paginationIncompleteReason: 'DATASET_PUSH_FAILED',
|
|
1382
|
+
},
|
|
1383
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1384
|
+
billableCount: reconciledItems.length,
|
|
1385
|
+
chargedEventCount: chargedSet.size,
|
|
1386
|
+
durationMs,
|
|
1387
|
+
requestCount: 1,
|
|
1388
|
+
}),
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
1391
|
+
for (const item of itemsToDeliver) {
|
|
1392
|
+
deliveredSet.add(item.creativeId);
|
|
1393
|
+
delete stagedItemsById[item.creativeId];
|
|
1394
|
+
reconciledItems.push(item);
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
deliveredCount = deliveredSet.size;
|
|
1398
|
+
await persistCheckpoint();
|
|
1399
|
+
reconciledInFlight = true;
|
|
1400
|
+
}
|
|
1401
|
+
} catch {
|
|
1402
|
+
reconciledInFlight = false;
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
if (!reconciledInFlight) {
|
|
1407
|
+
for (const id of inFlightArr) {
|
|
1408
|
+
if (!chargedSet.has(id) && !deliveredSet.has(id)) {
|
|
1409
|
+
unknownOutcomeSet.add(id);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
inFlightSet = new Set();
|
|
1413
|
+
inFlightIdempotencyKey = null;
|
|
1414
|
+
await persistCheckpoint();
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
if (unknownOutcomeSet.size > 0 && input?.strictBillingResume !== false) {
|
|
1419
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1420
|
+
return finalizeActorReturn({
|
|
1421
|
+
ok: false,
|
|
1422
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1423
|
+
reason: `Halted due to ${unknownOutcomeSet.size} unreconciled creative(s) in UNKNOWN_CHARGE_OUTCOME state`,
|
|
1424
|
+
billingMode,
|
|
1425
|
+
queryMeta: null,
|
|
1426
|
+
selectedEntities: [],
|
|
1427
|
+
items: reconciledItems,
|
|
1428
|
+
extractedCount: 0,
|
|
1429
|
+
deliveredCount,
|
|
1430
|
+
billableCount: reconciledItems.length,
|
|
1431
|
+
chargedEventCount: chargedSet.size,
|
|
1432
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1433
|
+
chargeReceipts: [],
|
|
1434
|
+
pagination: {
|
|
1435
|
+
nextPageToken: input?.pageToken || null,
|
|
1436
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1437
|
+
paginationIncompleteReason: 'UNKNOWN_CHARGE_OUTCOME',
|
|
1438
|
+
},
|
|
1439
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1440
|
+
billableCount: reconciledItems.length,
|
|
1441
|
+
chargedEventCount: chargedSet.size,
|
|
1442
|
+
durationMs,
|
|
1443
|
+
requestCount: 1,
|
|
1444
|
+
}),
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
const sleepFn =
|
|
1449
|
+
typeof hooks.sleepFn === 'function'
|
|
1450
|
+
? hooks.sleepFn
|
|
1451
|
+
: typeof input?.sleepFn === 'function'
|
|
1452
|
+
? input.sleepFn
|
|
1453
|
+
: undefined;
|
|
1454
|
+
const randomFn =
|
|
1455
|
+
typeof hooks.randomFn === 'function'
|
|
1456
|
+
? hooks.randomFn
|
|
1457
|
+
: typeof input?.randomFn === 'function'
|
|
1458
|
+
? input.randomFn
|
|
1459
|
+
: undefined;
|
|
1460
|
+
|
|
1461
|
+
// Step 1: Resolve Advertiser Entities
|
|
1462
|
+
const entityRes = await resolveAdvertiserEntities({
|
|
1463
|
+
query: input?.query,
|
|
1464
|
+
advertiserId: input?.advertiserId,
|
|
1465
|
+
advertiserName: input?.advertiserName,
|
|
1466
|
+
countryCode: input?.countryCode,
|
|
1467
|
+
includeSubsidiaries: input?.includeSubsidiaries,
|
|
1468
|
+
maxEntities: input?.maxEntities,
|
|
1469
|
+
fetchImpl: hooks.fetchImpl,
|
|
1470
|
+
nowFn: nowMs,
|
|
1471
|
+
sleepFn,
|
|
1472
|
+
randomFn,
|
|
1473
|
+
timeoutMs: input?.timeoutMs,
|
|
1474
|
+
maxRetries: input?.maxRetries,
|
|
1475
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1476
|
+
});
|
|
1477
|
+
|
|
1478
|
+
if (!entityRes.ok) {
|
|
1479
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1480
|
+
const isPreChargeDeadline =
|
|
1481
|
+
hasChargeHook &&
|
|
1482
|
+
nowMs() >= rpcDeadlineAtMs &&
|
|
1483
|
+
/deadline exceeded/i.test(String(entityRes.reason || ''));
|
|
1484
|
+
return finalizeActorReturn({
|
|
1485
|
+
ok: false,
|
|
1486
|
+
status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : entityRes.status,
|
|
1487
|
+
reason: entityRes.reason,
|
|
1488
|
+
billingMode,
|
|
1489
|
+
queryMeta: entityRes.queryMeta,
|
|
1490
|
+
selectedEntities: [],
|
|
1491
|
+
items: reconciledItems,
|
|
1492
|
+
extractedCount: 0,
|
|
1493
|
+
deliveredCount,
|
|
1494
|
+
billableCount: reconciledItems.length,
|
|
1495
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1496
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1497
|
+
chargeReceipts: [],
|
|
1498
|
+
pagination: {
|
|
1499
|
+
nextPageToken: input?.pageToken || null,
|
|
1500
|
+
paginationStatus: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_HAS_MORE' : entityRes.status,
|
|
1501
|
+
paginationIncompleteReason: isPreChargeDeadline
|
|
1502
|
+
? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
1503
|
+
: entityRes.reason || null,
|
|
1504
|
+
},
|
|
1505
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1506
|
+
billableCount: reconciledItems.length,
|
|
1507
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1508
|
+
durationMs,
|
|
1509
|
+
requestCount: Math.max(1, entityRes.rpcAttempts),
|
|
1510
|
+
}),
|
|
1511
|
+
});
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
if (entityRes.status === 'EMPTY_RESULT' || entityRes.selectedEntities.length === 0) {
|
|
1515
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1516
|
+
return finalizeActorReturn({
|
|
1517
|
+
ok: true,
|
|
1518
|
+
status: 'EMPTY_RESULT',
|
|
1519
|
+
billingMode,
|
|
1520
|
+
queryMeta: entityRes.queryMeta,
|
|
1521
|
+
selectedEntities: [],
|
|
1522
|
+
items: reconciledItems,
|
|
1523
|
+
extractedCount: 0,
|
|
1524
|
+
deliveredCount,
|
|
1525
|
+
billableCount: reconciledItems.length,
|
|
1526
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1527
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1528
|
+
chargeReceipts: [],
|
|
1529
|
+
pagination: {
|
|
1530
|
+
nextPageToken: null,
|
|
1531
|
+
paginationStatus: 'EXHAUSTED',
|
|
1532
|
+
paginationIncompleteReason: null,
|
|
1533
|
+
},
|
|
1534
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1535
|
+
billableCount: reconciledItems.length,
|
|
1536
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1537
|
+
durationMs,
|
|
1538
|
+
requestCount: Math.max(1, entityRes.rpcAttempts),
|
|
1539
|
+
}),
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// Step 2: Fetch Creatives across selected verified advertiser entities
|
|
1544
|
+
const advertiserIds = entityRes.selectedEntities.map((e) => e.advertiserId);
|
|
1545
|
+
const creativesRes = await fetchAdvertiserCreatives({
|
|
1546
|
+
advertiserIds,
|
|
1547
|
+
entities: entityRes.selectedEntities,
|
|
1548
|
+
maxResults: input?.maxResults,
|
|
1549
|
+
maxPages: input?.maxPages,
|
|
1550
|
+
pageSize: input?.pageSize,
|
|
1551
|
+
pageToken: input?.pageToken,
|
|
1552
|
+
countryCode: input?.countryCode,
|
|
1553
|
+
formatFilter: input?.formatFilter,
|
|
1554
|
+
fetchImpl: hooks.fetchImpl,
|
|
1555
|
+
nowFn: nowMs,
|
|
1556
|
+
sleepFn,
|
|
1557
|
+
randomFn,
|
|
1558
|
+
timeoutMs: input?.timeoutMs,
|
|
1559
|
+
maxRetries: input?.maxRetries,
|
|
1560
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1561
|
+
});
|
|
1562
|
+
|
|
1563
|
+
let totalRpcRequests = entityRes.rpcAttempts + creativesRes.rpcAttempts;
|
|
1564
|
+
|
|
1565
|
+
if (!creativesRes.ok) {
|
|
1566
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1567
|
+
const isPreChargeDeadline =
|
|
1568
|
+
hasChargeHook &&
|
|
1569
|
+
nowMs() >= rpcDeadlineAtMs &&
|
|
1570
|
+
/deadline exceeded/i.test(String(creativesRes.reason || ''));
|
|
1571
|
+
return finalizeActorReturn({
|
|
1572
|
+
ok: false,
|
|
1573
|
+
status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : creativesRes.status,
|
|
1574
|
+
reason: creativesRes.reason,
|
|
1575
|
+
billingMode,
|
|
1576
|
+
queryMeta: entityRes.queryMeta,
|
|
1577
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1578
|
+
items: reconciledItems,
|
|
1579
|
+
extractedCount: 0,
|
|
1580
|
+
deliveredCount,
|
|
1581
|
+
billableCount: reconciledItems.length,
|
|
1582
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1583
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1584
|
+
chargeReceipts: [],
|
|
1585
|
+
pagination: {
|
|
1586
|
+
nextPageToken: input?.pageToken || null,
|
|
1587
|
+
paginationStatus: isPreChargeDeadline
|
|
1588
|
+
? 'DEADLINE_EXCEEDED_HAS_MORE'
|
|
1589
|
+
: creativesRes.paginationStatus || creativesRes.status,
|
|
1590
|
+
paginationIncompleteReason: isPreChargeDeadline
|
|
1591
|
+
? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
1592
|
+
: creativesRes.paginationIncompleteReason || creativesRes.reason || null,
|
|
1593
|
+
},
|
|
1594
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1595
|
+
billableCount: reconciledItems.length,
|
|
1596
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1597
|
+
durationMs,
|
|
1598
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1599
|
+
}),
|
|
1600
|
+
});
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
const maxResultsCap = Math.min(200, Math.max(1, Number(input?.maxResults) || 40));
|
|
1604
|
+
|
|
1605
|
+
// Step 3: Strict Deduplication & Validation Gate before emitting Pay-per-Event charges
|
|
1606
|
+
const uniqueExtractedItems = [];
|
|
1607
|
+
const seenInRunIds = new Set();
|
|
1608
|
+
|
|
1609
|
+
for (const item of creativesRes.creatives) {
|
|
1610
|
+
if (
|
|
1611
|
+
!item ||
|
|
1612
|
+
!ADVERTISER_ID_RE.test(String(item.advertiserId || '')) ||
|
|
1613
|
+
!CREATIVE_ID_RE.test(String(item.creativeId || ''))
|
|
1614
|
+
) {
|
|
1615
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1616
|
+
return finalizeActorReturn({
|
|
1617
|
+
ok: false,
|
|
1618
|
+
status: 'SCHEMA_DRIFT_HALT',
|
|
1619
|
+
reason: 'Normalized creative record failed AR.../CR... identifier gate prior to billing',
|
|
1620
|
+
billingMode,
|
|
1621
|
+
queryMeta: entityRes.queryMeta,
|
|
1622
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1623
|
+
items: reconciledItems,
|
|
1624
|
+
extractedCount: 0,
|
|
1625
|
+
deliveredCount,
|
|
1626
|
+
billableCount: reconciledItems.length,
|
|
1627
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1628
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1629
|
+
chargeReceipts: [],
|
|
1630
|
+
pagination: {
|
|
1631
|
+
nextPageToken: null,
|
|
1632
|
+
paginationStatus: 'SCHEMA_DRIFT_HALT',
|
|
1633
|
+
paginationIncompleteReason: 'INVALID_CREATIVE_IDENTIFIER',
|
|
1634
|
+
},
|
|
1635
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1636
|
+
billableCount: reconciledItems.length,
|
|
1637
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1638
|
+
durationMs,
|
|
1639
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1640
|
+
}),
|
|
1641
|
+
});
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
if (!seenInRunIds.has(item.creativeId)) {
|
|
1645
|
+
seenInRunIds.add(item.creativeId);
|
|
1646
|
+
uniqueExtractedItems.push(item);
|
|
1647
|
+
if (uniqueExtractedItems.length >= maxResultsCap) {
|
|
1648
|
+
break;
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
// Optional Step 3b: Enrich up to 10 creatives (default 5) via LookupService/GetCreativeById when includeCreativeDetails === true
|
|
1654
|
+
if (input?.includeCreativeDetails === true && uniqueExtractedItems.length > 0) {
|
|
1655
|
+
const requestedEnrichCap = Number(input?.maxEnrichItems ?? 5);
|
|
1656
|
+
const enrichCap = Math.min(
|
|
1657
|
+
10,
|
|
1658
|
+
Math.max(1, Number.isFinite(requestedEnrichCap) ? Math.floor(requestedEnrichCap) : 5),
|
|
1659
|
+
uniqueExtractedItems.length
|
|
1660
|
+
);
|
|
1661
|
+
let rateLimitedInEnrich = false;
|
|
1662
|
+
for (let i = 0; i < enrichCap; i += 1) {
|
|
1663
|
+
const targetItem = uniqueExtractedItems[i];
|
|
1664
|
+
if (rateLimitedInEnrich) {
|
|
1665
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1666
|
+
'NOT_ATTEMPTED_RATE_LIMITED'
|
|
1667
|
+
);
|
|
1668
|
+
continue;
|
|
1669
|
+
}
|
|
1670
|
+
if (nowMs() >= rpcDeadlineAtMs) {
|
|
1671
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1672
|
+
'NOT_ATTEMPTED_DEADLINE'
|
|
1673
|
+
);
|
|
1674
|
+
continue;
|
|
1675
|
+
}
|
|
1676
|
+
const detailRes = await getCreativeDetailsById({
|
|
1677
|
+
advertiserId: targetItem.advertiserId,
|
|
1678
|
+
creativeId: targetItem.creativeId,
|
|
1679
|
+
includeAdCopy: Boolean(input?.includeAdCopy),
|
|
1680
|
+
fallbackPreviewUrl: targetItem.previewUrl,
|
|
1681
|
+
fetchImpl: hooks.fetchImpl,
|
|
1682
|
+
nowFn: nowMs,
|
|
1683
|
+
sleepFn,
|
|
1684
|
+
randomFn,
|
|
1685
|
+
timeoutMs: input?.timeoutMs,
|
|
1686
|
+
maxRetries: input?.maxRetries !== undefined ? input.maxRetries : 1,
|
|
1687
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1688
|
+
});
|
|
1689
|
+
totalRpcRequests += detailRes.rpcAttempts;
|
|
1690
|
+
if (detailRes.status === 'UPSTREAM_RATE_LIMITED') {
|
|
1691
|
+
rateLimitedInEnrich = true;
|
|
1692
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1693
|
+
'NOT_ATTEMPTED_RATE_LIMITED'
|
|
1694
|
+
);
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
if (detailRes.ok && detailRes.detail) {
|
|
1698
|
+
targetItem.creativeDetail = detailRes.detail;
|
|
1699
|
+
if (detailRes.detail.untrustedAdContent) {
|
|
1700
|
+
targetItem.untrustedAdContent = detailRes.detail.untrustedAdContent;
|
|
1701
|
+
if (
|
|
1702
|
+
detailRes.detail.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
|
|
1703
|
+
detailRes.detail.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
|
|
1704
|
+
) {
|
|
1705
|
+
rateLimitedInEnrich = true;
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
} else if (detailRes.ok && detailRes.untrustedAdContent) {
|
|
1709
|
+
targetItem.untrustedAdContent = detailRes.untrustedAdContent;
|
|
1710
|
+
if (
|
|
1711
|
+
detailRes.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
|
|
1712
|
+
detailRes.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
|
|
1713
|
+
) {
|
|
1714
|
+
rateLimitedInEnrich = true;
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
const extractedCount = uniqueExtractedItems.length;
|
|
1721
|
+
|
|
1722
|
+
// Filter out items already charged, delivered, or marked unknown in prior checkpoint
|
|
1723
|
+
const newBillableCandidates = uniqueExtractedItems.filter(
|
|
1724
|
+
(item) =>
|
|
1725
|
+
!chargedSet.has(item.creativeId) &&
|
|
1726
|
+
!deliveredSet.has(item.creativeId) &&
|
|
1727
|
+
!unknownOutcomeSet.has(item.creativeId)
|
|
1728
|
+
);
|
|
1729
|
+
|
|
1730
|
+
if (newBillableCandidates.length === 0 && reconciledItems.length === 0) {
|
|
1731
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1732
|
+
return finalizeActorReturn({
|
|
1733
|
+
ok: true,
|
|
1734
|
+
status: 'EMPTY_RESULT',
|
|
1735
|
+
billingMode,
|
|
1736
|
+
queryMeta: entityRes.queryMeta,
|
|
1737
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1738
|
+
items: [],
|
|
1739
|
+
extractedCount,
|
|
1740
|
+
deliveredCount,
|
|
1741
|
+
billableCount: 0,
|
|
1742
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1743
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1744
|
+
chargeReceipts: [],
|
|
1745
|
+
pagination: {
|
|
1746
|
+
nextPageToken: creativesRes.nextPageToken,
|
|
1747
|
+
paginationStatus: creativesRes.paginationStatus,
|
|
1748
|
+
paginationIncompleteReason: creativesRes.paginationIncompleteReason,
|
|
1749
|
+
pagesFetched: creativesRes.pagesFetched,
|
|
1750
|
+
},
|
|
1751
|
+
cursorResumeContext: creativesRes.cursorResumeContext,
|
|
1752
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1753
|
+
billableCount: 0,
|
|
1754
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1755
|
+
durationMs,
|
|
1756
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1757
|
+
unitPriceUsdc: hooks?.unitPriceUsdc ?? ACTOR_PRICING_CONTRACT.priceUsdc,
|
|
1758
|
+
actorStartFeeUsdc: hooks?.actorStartFeeUsdc,
|
|
1759
|
+
}),
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// Step 4: Chunked Dataset Push (25 items/chunk) & Crash-Safe Event Charging
|
|
1764
|
+
const chunkSize = ACTOR_PRICING_CONTRACT.chunkSize;
|
|
1765
|
+
const chargeReceipts = [];
|
|
1766
|
+
const newlyDeliveredItems = [];
|
|
1767
|
+
let chargedEventCount = hasChargeHook ? chargedSet.size : 0;
|
|
1768
|
+
let finalStatus = 'OK';
|
|
1769
|
+
let finalOk = true;
|
|
1770
|
+
let haltReason = undefined;
|
|
1771
|
+
const runtimeWarnings = Array.isArray(creativesRes.warnings) ? [...creativesRes.warnings] : [];
|
|
1772
|
+
const persistCheckpointPostCharge = async () => {
|
|
1773
|
+
try {
|
|
1774
|
+
await persistCheckpoint();
|
|
1775
|
+
} catch (cpErr) {
|
|
1776
|
+
runtimeWarnings.push(
|
|
1777
|
+
`Post-charge checkpoint write warning: ${cpErr?.message || String(cpErr)}`
|
|
1778
|
+
);
|
|
1779
|
+
}
|
|
1780
|
+
};
|
|
1781
|
+
|
|
1782
|
+
let lastResolvedUnitPriceUsdc = hooks?.unitPriceUsdc ?? ACTOR_PRICING_CONTRACT.priceUsdc;
|
|
1783
|
+
for (let offset = 0; offset < newBillableCandidates.length; offset += chunkSize) {
|
|
1784
|
+
const effectiveChunkDeadlineMs = hasChargeHook
|
|
1785
|
+
? deadlineAtMs - step4ReserveMs
|
|
1786
|
+
: deadlineAtMs;
|
|
1787
|
+
if (nowMs() >= effectiveChunkDeadlineMs) {
|
|
1788
|
+
finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
|
|
1789
|
+
finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
|
|
1790
|
+
haltReason = 'Run deadline reserve reached prior to charging next chunk; halted cleanly before billing';
|
|
1791
|
+
break;
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
const unchargedChunk = newBillableCandidates
|
|
1795
|
+
.slice(offset, offset + chunkSize)
|
|
1796
|
+
.filter(
|
|
1797
|
+
(c) =>
|
|
1798
|
+
!chargedSet.has(c.creativeId) &&
|
|
1799
|
+
!deliveredSet.has(c.creativeId) &&
|
|
1800
|
+
!unknownOutcomeSet.has(c.creativeId)
|
|
1801
|
+
);
|
|
1802
|
+
|
|
1803
|
+
if (unchargedChunk.length === 0) {
|
|
1804
|
+
continue;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
const chunkIds = unchargedChunk.map((c) => c.creativeId);
|
|
1808
|
+
const idempotencyKey = computeChunkIdempotencyKey(runId, chunkIds);
|
|
1809
|
+
|
|
1810
|
+
if (!hasChargeHook) {
|
|
1811
|
+
// Local / MCP unbilled execution mode: deliver items with chargedEventCount = 0
|
|
1812
|
+
const pushRes = await pushDataWithRetry(unchargedChunk);
|
|
1813
|
+
if (!pushRes.ok) {
|
|
1814
|
+
await persistCheckpointPostCharge();
|
|
1815
|
+
finalOk = false;
|
|
1816
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1817
|
+
haltReason = `Dataset pushData failed in local mode: ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1818
|
+
break;
|
|
1819
|
+
}
|
|
1820
|
+
for (const c of unchargedChunk) {
|
|
1821
|
+
deliveredSet.add(c.creativeId);
|
|
1822
|
+
delete stagedItemsById[c.creativeId];
|
|
1823
|
+
newlyDeliveredItems.push(c);
|
|
1824
|
+
}
|
|
1825
|
+
deliveredCount = deliveredSet.size;
|
|
1826
|
+
await persistCheckpointPostCharge();
|
|
1827
|
+
continue;
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
// Step 4.1: Persist inFlightIds, inFlightIdempotencyKey, and stagedItemsById BEFORE invoking chargeEvent
|
|
1831
|
+
inFlightSet = new Set(chunkIds);
|
|
1832
|
+
inFlightIdempotencyKey = idempotencyKey;
|
|
1833
|
+
for (const c of unchargedChunk) {
|
|
1834
|
+
stagedItemsById[c.creativeId] = c;
|
|
1835
|
+
}
|
|
1836
|
+
try {
|
|
1837
|
+
await persistCheckpoint();
|
|
1838
|
+
} catch (preCpErr) {
|
|
1839
|
+
inFlightSet = new Set();
|
|
1840
|
+
inFlightIdempotencyKey = null;
|
|
1841
|
+
finalOk = false;
|
|
1842
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1843
|
+
haltReason = `Pre-charge checkpoint write failed before billing: ${preCpErr?.message || String(preCpErr)}`;
|
|
1844
|
+
break;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
if (nowMs() >= effectiveChunkDeadlineMs) {
|
|
1848
|
+
inFlightSet = new Set();
|
|
1849
|
+
inFlightIdempotencyKey = null;
|
|
1850
|
+
await persistCheckpointPostCharge();
|
|
1851
|
+
finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
|
|
1852
|
+
finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
|
|
1853
|
+
haltReason =
|
|
1854
|
+
'Run deadline reserve reached immediately after pre-charge checkpoint write; cleared inFlightIds and halted before billing';
|
|
1855
|
+
break;
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
const receipt = {
|
|
1859
|
+
idempotencyKey,
|
|
1860
|
+
eventName: ACTOR_PRICING_CONTRACT.eventName,
|
|
1861
|
+
count: unchargedChunk.length,
|
|
1862
|
+
unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
|
|
1863
|
+
chunkAmountUsdc: Number(
|
|
1864
|
+
(unchargedChunk.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
|
|
1865
|
+
),
|
|
1866
|
+
creativeIds: chunkIds,
|
|
1867
|
+
deadlineAtMs,
|
|
1868
|
+
};
|
|
1869
|
+
|
|
1870
|
+
let chargeResult;
|
|
1871
|
+
try {
|
|
1872
|
+
chargeResult = await hooks.chargeEvent(receipt);
|
|
1873
|
+
} catch (chargeErr) {
|
|
1874
|
+
for (const id of chunkIds) {
|
|
1875
|
+
unknownOutcomeSet.add(id);
|
|
1876
|
+
}
|
|
1877
|
+
inFlightSet = new Set();
|
|
1878
|
+
inFlightIdempotencyKey = null;
|
|
1879
|
+
await persistCheckpointPostCharge();
|
|
1880
|
+
finalOk = false;
|
|
1881
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1882
|
+
haltReason = `chargeEvent threw error: ${chargeErr?.message || String(chargeErr)}`;
|
|
1883
|
+
break;
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
// Step 4.2: Strict ChargeResult validation (never fallback to ?? unchargedChunk.length)
|
|
1887
|
+
const chargedCount = chargeResult?.chargedCount;
|
|
1888
|
+
const eventChargeLimitReached = Boolean(chargeResult?.eventChargeLimitReached);
|
|
1889
|
+
const resolvedReceiptEventName =
|
|
1890
|
+
typeof chargeResult?.eventName === 'string' && chargeResult.eventName
|
|
1891
|
+
? chargeResult.eventName
|
|
1892
|
+
: receipt.eventName;
|
|
1893
|
+
const resolvedReceiptUnitPriceUsdc =
|
|
1894
|
+
Number.isFinite(Number(chargeResult?.unitPriceUsdc)) && Number(chargeResult.unitPriceUsdc) > 0
|
|
1895
|
+
? Number(chargeResult.unitPriceUsdc)
|
|
1896
|
+
: receipt.unitPriceUsdc;
|
|
1897
|
+
lastResolvedUnitPriceUsdc = resolvedReceiptUnitPriceUsdc;
|
|
1898
|
+
const resolvedReceiptIdempotencyKey =
|
|
1899
|
+
typeof chargeResult?.idempotencyKey === 'string' && chargeResult.idempotencyKey
|
|
1900
|
+
? chargeResult.idempotencyKey
|
|
1901
|
+
: receipt.idempotencyKey;
|
|
1902
|
+
const buildFinalReceipt = (actualChargedCount, extra = {}) => ({
|
|
1903
|
+
...receipt,
|
|
1904
|
+
idempotencyKey: resolvedReceiptIdempotencyKey,
|
|
1905
|
+
eventName: resolvedReceiptEventName,
|
|
1906
|
+
unitPriceUsdc: resolvedReceiptUnitPriceUsdc,
|
|
1907
|
+
chunkAmountUsdc: Number((actualChargedCount * resolvedReceiptUnitPriceUsdc).toFixed(6)),
|
|
1908
|
+
chargedCount: actualChargedCount,
|
|
1909
|
+
...extra,
|
|
1910
|
+
});
|
|
1911
|
+
|
|
1912
|
+
if (
|
|
1913
|
+
!chargeResult ||
|
|
1914
|
+
typeof chargeResult !== 'object' ||
|
|
1915
|
+
!Number.isInteger(chargedCount) ||
|
|
1916
|
+
chargedCount < 0 ||
|
|
1917
|
+
chargedCount > unchargedChunk.length
|
|
1918
|
+
) {
|
|
1919
|
+
for (const id of chunkIds) {
|
|
1920
|
+
unknownOutcomeSet.add(id);
|
|
1921
|
+
}
|
|
1922
|
+
inFlightSet = new Set();
|
|
1923
|
+
await persistCheckpointPostCharge();
|
|
1924
|
+
finalOk = false;
|
|
1925
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1926
|
+
haltReason =
|
|
1927
|
+
'chargeEvent returned invalid or missing integer chargedCount (transitioned chunk to UNKNOWN_CHARGE_OUTCOME)';
|
|
1928
|
+
break;
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
// Step 4.3: Branch on validated chargedCount
|
|
1932
|
+
if (chargedCount === unchargedChunk.length) {
|
|
1933
|
+
// Full chunk charged: persist chargedIds immediately BEFORE pushData (best-effort so KVS glitch never blocks delivery)
|
|
1934
|
+
for (const id of chunkIds) {
|
|
1935
|
+
chargedSet.add(id);
|
|
1936
|
+
}
|
|
1937
|
+
inFlightSet = new Set();
|
|
1938
|
+
chargedEventCount = chargedSet.size;
|
|
1939
|
+
await persistCheckpointPostCharge();
|
|
1940
|
+
|
|
1941
|
+
const pushRes = await pushDataWithRetry(unchargedChunk);
|
|
1942
|
+
if (!pushRes.ok) {
|
|
1943
|
+
chargeReceipts.push(buildFinalReceipt(chargedCount, { eventChargeLimitReached }));
|
|
1944
|
+
finalOk = false;
|
|
1945
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1946
|
+
haltReason = `Dataset pushData failed after charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1947
|
+
break;
|
|
1948
|
+
}
|
|
1949
|
+
for (const c of unchargedChunk) {
|
|
1950
|
+
deliveredSet.add(c.creativeId);
|
|
1951
|
+
delete stagedItemsById[c.creativeId];
|
|
1952
|
+
newlyDeliveredItems.push(c);
|
|
1953
|
+
}
|
|
1954
|
+
deliveredCount = deliveredSet.size;
|
|
1955
|
+
await persistCheckpointPostCharge();
|
|
1956
|
+
|
|
1957
|
+
chargeReceipts.push(buildFinalReceipt(chargedCount, { eventChargeLimitReached }));
|
|
1958
|
+
|
|
1959
|
+
if (eventChargeLimitReached) {
|
|
1960
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
1961
|
+
break;
|
|
1962
|
+
}
|
|
1963
|
+
} else if (chargedCount === 0) {
|
|
1964
|
+
inFlightSet = new Set();
|
|
1965
|
+
if (chargeResult?.unbilledOwnerTestBypass === true) {
|
|
1966
|
+
billingMode = 'UNBILLED_OWNER_TEST';
|
|
1967
|
+
await persistCheckpointPostCharge();
|
|
1968
|
+
const pushRes = await pushDataWithRetry(unchargedChunk);
|
|
1969
|
+
if (!pushRes.ok) {
|
|
1970
|
+
finalOk = false;
|
|
1971
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1972
|
+
haltReason = `Dataset pushData failed in UNBILLED_OWNER_TEST mode: ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1973
|
+
break;
|
|
1974
|
+
}
|
|
1975
|
+
for (const c of unchargedChunk) {
|
|
1976
|
+
deliveredSet.add(c.creativeId);
|
|
1977
|
+
delete stagedItemsById[c.creativeId];
|
|
1978
|
+
newlyDeliveredItems.push(c);
|
|
1979
|
+
}
|
|
1980
|
+
deliveredCount = deliveredSet.size;
|
|
1981
|
+
await persistCheckpointPostCharge();
|
|
1982
|
+
chargeReceipts.push(
|
|
1983
|
+
buildFinalReceipt(0, {
|
|
1984
|
+
eventChargeLimitReached: false,
|
|
1985
|
+
unbilledOwnerTestBypass: true,
|
|
1986
|
+
})
|
|
1987
|
+
);
|
|
1988
|
+
continue;
|
|
1989
|
+
}
|
|
1990
|
+
await persistCheckpointPostCharge();
|
|
1991
|
+
chargeReceipts.push(buildFinalReceipt(0, { eventChargeLimitReached }));
|
|
1992
|
+
if (eventChargeLimitReached) {
|
|
1993
|
+
finalOk = true;
|
|
1994
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
1995
|
+
} else {
|
|
1996
|
+
finalOk = false;
|
|
1997
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1998
|
+
haltReason = 'chargeEvent returned chargedCount = 0 without eventChargeLimitReached';
|
|
1999
|
+
}
|
|
2000
|
+
break;
|
|
2001
|
+
} else {
|
|
2002
|
+
// Partial chunk charge (0 < chargedCount < unchargedChunk.length)
|
|
2003
|
+
const hasExplicitIdsArray = Array.isArray(chargeResult.chargedCreativeIds);
|
|
2004
|
+
const explicitChargedIds = hasExplicitIdsArray
|
|
2005
|
+
? Array.from(
|
|
2006
|
+
new Set(chargeResult.chargedCreativeIds.filter((id) => chunkIds.includes(id)))
|
|
2007
|
+
)
|
|
2008
|
+
: null;
|
|
2009
|
+
const isValidExplicitChargedIds = Boolean(
|
|
2010
|
+
explicitChargedIds &&
|
|
2011
|
+
explicitChargedIds.length === chargedCount &&
|
|
2012
|
+
chargeResult.chargedCreativeIds.length === chargedCount
|
|
2013
|
+
);
|
|
2014
|
+
|
|
2015
|
+
if (isValidExplicitChargedIds) {
|
|
2016
|
+
for (const id of explicitChargedIds) {
|
|
2017
|
+
chargedSet.add(id);
|
|
2018
|
+
}
|
|
2019
|
+
inFlightSet = new Set();
|
|
2020
|
+
chargedEventCount = chargedSet.size;
|
|
2021
|
+
await persistCheckpointPostCharge();
|
|
2022
|
+
|
|
2023
|
+
const paidSubset = unchargedChunk.filter((c) => explicitChargedIds.includes(c.creativeId));
|
|
2024
|
+
const pushRes = await pushDataWithRetry(paidSubset);
|
|
2025
|
+
if (!pushRes.ok) {
|
|
2026
|
+
chargeReceipts.push(
|
|
2027
|
+
buildFinalReceipt(chargedCount, {
|
|
2028
|
+
chargedCreativeIds: explicitChargedIds,
|
|
2029
|
+
eventChargeLimitReached,
|
|
2030
|
+
})
|
|
2031
|
+
);
|
|
2032
|
+
finalOk = false;
|
|
2033
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
2034
|
+
haltReason = `Dataset pushData failed after partial charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
2035
|
+
break;
|
|
2036
|
+
}
|
|
2037
|
+
for (const c of paidSubset) {
|
|
2038
|
+
deliveredSet.add(c.creativeId);
|
|
2039
|
+
delete stagedItemsById[c.creativeId];
|
|
2040
|
+
newlyDeliveredItems.push(c);
|
|
2041
|
+
}
|
|
2042
|
+
deliveredCount = deliveredSet.size;
|
|
2043
|
+
await persistCheckpointPostCharge();
|
|
2044
|
+
|
|
2045
|
+
chargeReceipts.push(
|
|
2046
|
+
buildFinalReceipt(chargedCount, {
|
|
2047
|
+
chargedCreativeIds: explicitChargedIds,
|
|
2048
|
+
eventChargeLimitReached,
|
|
2049
|
+
})
|
|
2050
|
+
);
|
|
2051
|
+
|
|
2052
|
+
finalOk = eventChargeLimitReached;
|
|
2053
|
+
finalStatus = eventChargeLimitReached ? 'CHARGE_LIMIT_REACHED' : 'PARTIAL_BILLING_HALT';
|
|
2054
|
+
if (!eventChargeLimitReached) {
|
|
2055
|
+
haltReason = `Partial chunk charge (${chargedCount}/${unchargedChunk.length}) without charge limit flag`;
|
|
2056
|
+
}
|
|
2057
|
+
break;
|
|
2058
|
+
} else if (!hasExplicitIdsArray && eventChargeLimitReached) {
|
|
2059
|
+
// Standard Apify SDK ChargeResult when hitting user's max charge ceiling:
|
|
2060
|
+
// Attribute charge deterministically to the ordered prefix `unchargedChunk.slice(0, chargedCount)`
|
|
2061
|
+
// so the customer receives every item they paid for, and leave the remaining tail uncharged.
|
|
2062
|
+
const paidPrefix = unchargedChunk.slice(0, chargedCount);
|
|
2063
|
+
const paidPrefixIds = paidPrefix.map((c) => c.creativeId);
|
|
2064
|
+
for (const id of paidPrefixIds) {
|
|
2065
|
+
chargedSet.add(id);
|
|
2066
|
+
}
|
|
2067
|
+
inFlightSet = new Set();
|
|
2068
|
+
chargedEventCount = chargedSet.size;
|
|
2069
|
+
await persistCheckpointPostCharge();
|
|
2070
|
+
|
|
2071
|
+
const pushRes = await pushDataWithRetry(paidPrefix);
|
|
2072
|
+
if (!pushRes.ok) {
|
|
2073
|
+
chargeReceipts.push(
|
|
2074
|
+
buildFinalReceipt(chargedCount, {
|
|
2075
|
+
chargedCreativeIds: paidPrefixIds,
|
|
2076
|
+
eventChargeLimitReached: true,
|
|
2077
|
+
})
|
|
2078
|
+
);
|
|
2079
|
+
finalOk = false;
|
|
2080
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
2081
|
+
haltReason = `Dataset pushData failed after charge limit prefix (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
2082
|
+
break;
|
|
2083
|
+
}
|
|
2084
|
+
for (const c of paidPrefix) {
|
|
2085
|
+
deliveredSet.add(c.creativeId);
|
|
2086
|
+
delete stagedItemsById[c.creativeId];
|
|
2087
|
+
newlyDeliveredItems.push(c);
|
|
2088
|
+
}
|
|
2089
|
+
deliveredCount = deliveredSet.size;
|
|
2090
|
+
await persistCheckpointPostCharge();
|
|
2091
|
+
|
|
2092
|
+
chargeReceipts.push(
|
|
2093
|
+
buildFinalReceipt(chargedCount, {
|
|
2094
|
+
chargedCreativeIds: paidPrefixIds,
|
|
2095
|
+
eventChargeLimitReached: true,
|
|
2096
|
+
})
|
|
2097
|
+
);
|
|
2098
|
+
|
|
2099
|
+
finalOk = true;
|
|
2100
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
2101
|
+
break;
|
|
2102
|
+
} else {
|
|
2103
|
+
// Unexpected partial count without eventChargeLimitReached and without explicit IDs ->
|
|
2104
|
+
// move chunk to unknownOutcomeIds and halt fail-closed
|
|
2105
|
+
for (const id of chunkIds) {
|
|
2106
|
+
unknownOutcomeSet.add(id);
|
|
2107
|
+
}
|
|
2108
|
+
inFlightSet = new Set();
|
|
2109
|
+
chargedEventCount = chargedSet.size + chargedCount;
|
|
2110
|
+
await persistCheckpointPostCharge();
|
|
2111
|
+
|
|
2112
|
+
chargeReceipts.push(
|
|
2113
|
+
buildFinalReceipt(chargedCount, {
|
|
2114
|
+
eventChargeLimitReached: false,
|
|
2115
|
+
})
|
|
2116
|
+
);
|
|
2117
|
+
|
|
2118
|
+
finalOk = false;
|
|
2119
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
2120
|
+
haltReason = `Ambiguous partial chunk charge (${chargedCount}/${unchargedChunk.length}) without chargedCreativeIds attribution or eventChargeLimitReached`;
|
|
2121
|
+
break;
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
// Step 5: Rewind pagination cursor if billing halted or reached charge limit before delivering all extracted items
|
|
2127
|
+
let effectiveNextPageToken = creativesRes.nextPageToken;
|
|
2128
|
+
let effectivePaginationStatus = creativesRes.paginationStatus;
|
|
2129
|
+
let effectiveIncompleteReason = creativesRes.paginationIncompleteReason;
|
|
2130
|
+
const undeliveredCount = Math.max(0, newBillableCandidates.length - newlyDeliveredItems.length);
|
|
2131
|
+
|
|
2132
|
+
if (undeliveredCount > 0) {
|
|
2133
|
+
const firstUndeliveredItem = newBillableCandidates.find((c) => !deliveredSet.has(c.creativeId));
|
|
2134
|
+
const firstUndeliveredExtractedIdx = Array.isArray(creativesRes.creatives)
|
|
2135
|
+
? creativesRes.creatives.findIndex((c) => c.creativeId === firstUndeliveredItem?.creativeId)
|
|
2136
|
+
: -1;
|
|
2137
|
+
const effectiveDeliveredPrefixLen =
|
|
2138
|
+
firstUndeliveredExtractedIdx >= 0 ? firstUndeliveredExtractedIdx : newlyDeliveredItems.length;
|
|
2139
|
+
effectiveNextPageToken = computeRewoundCursorForDeliveredPrefix(
|
|
2140
|
+
creativesRes.cursorResumeContext,
|
|
2141
|
+
effectiveDeliveredPrefixLen,
|
|
2142
|
+
creativesRes.nextPageToken
|
|
2143
|
+
);
|
|
2144
|
+
effectivePaginationStatus =
|
|
2145
|
+
finalStatus === 'CHARGE_LIMIT_REACHED'
|
|
2146
|
+
? 'CHARGE_LIMIT_REACHED_HAS_MORE'
|
|
2147
|
+
: finalStatus === 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
2148
|
+
? 'DEADLINE_EXCEEDED_HAS_MORE'
|
|
2149
|
+
: 'PARTIAL_BILLING_HALT';
|
|
2150
|
+
effectiveIncompleteReason = finalStatus;
|
|
2151
|
+
} else if (finalStatus === 'CHARGE_LIMIT_REACHED' && effectiveNextPageToken) {
|
|
2152
|
+
effectivePaginationStatus = 'CHARGE_LIMIT_REACHED_HAS_MORE';
|
|
2153
|
+
effectiveIncompleteReason = finalStatus;
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
const paginationMeta = {
|
|
2157
|
+
nextPageToken: effectiveNextPageToken,
|
|
2158
|
+
paginationStatus: effectivePaginationStatus,
|
|
2159
|
+
paginationIncompleteReason: effectiveIncompleteReason,
|
|
2160
|
+
pagesFetched: creativesRes.pagesFetched,
|
|
2161
|
+
...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
|
|
2162
|
+
...(undeliveredCount > 0 ? { undeliveredCount } : {}),
|
|
2163
|
+
};
|
|
2164
|
+
|
|
2165
|
+
// Merge reconciled items + newly delivered items with strict creativeId deduplication
|
|
2166
|
+
const allOutputItems = [];
|
|
2167
|
+
const outputSeenIds = new Set();
|
|
2168
|
+
for (const item of [...reconciledItems, ...newlyDeliveredItems]) {
|
|
2169
|
+
if (item?.creativeId && !outputSeenIds.has(item.creativeId)) {
|
|
2170
|
+
outputSeenIds.add(item.creativeId);
|
|
2171
|
+
allOutputItems.push(item);
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
const simulatedBillableCount = hasChargeHook ? chargedEventCount : allOutputItems.length;
|
|
2176
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
2177
|
+
const telemetry = computeRunEconomicsTelemetry({
|
|
2178
|
+
billableCount: simulatedBillableCount,
|
|
2179
|
+
chargedEventCount,
|
|
2180
|
+
durationMs,
|
|
2181
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
2182
|
+
unitPriceUsdc: lastResolvedUnitPriceUsdc,
|
|
2183
|
+
actorStartFeeUsdc: hooks?.actorStartFeeUsdc,
|
|
2184
|
+
});
|
|
2185
|
+
|
|
2186
|
+
return finalizeActorReturn({
|
|
2187
|
+
ok: finalOk,
|
|
2188
|
+
status: finalStatus,
|
|
2189
|
+
reason: haltReason,
|
|
2190
|
+
billingMode,
|
|
2191
|
+
queryMeta: entityRes.queryMeta,
|
|
2192
|
+
selectedEntities: entityRes.selectedEntities,
|
|
2193
|
+
items: allOutputItems,
|
|
2194
|
+
extractedCount,
|
|
2195
|
+
deliveredCount,
|
|
2196
|
+
billableCount: simulatedBillableCount,
|
|
2197
|
+
chargedEventCount,
|
|
2198
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
2199
|
+
chargeReceipts,
|
|
2200
|
+
pagination: paginationMeta,
|
|
2201
|
+
...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
|
|
2202
|
+
cursorResumeContext: creativesRes.cursorResumeContext,
|
|
2203
|
+
telemetry,
|
|
2204
|
+
});
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
export const scrapeGoogleAdsTransparency = runGoogleAdsTransparencyActor;
|
|
2208
|
+
|