@revclamp/google-ads-transparency-mcp-actor 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.actor/actor.json +17 -0
- package/.actor/input_schema.json +76 -0
- package/.well-known/glama.json +6 -0
- package/LICENSE +21 -0
- package/README.md +39 -2
- package/actor_config.json +29 -0
- package/assets/icon-180.png +0 -0
- package/assets/icon-32.png +0 -0
- package/assets/icon.svg +26 -0
- package/assets/playground.css +468 -0
- package/assets/playground.js +795 -0
- package/glama.json +6 -0
- package/index.mjs +222 -0
- package/package.json +38 -4
- package/server.json +36 -0
- package/src/apify_actor.mjs +1714 -0
- package/src/egress_guard.mjs +69 -0
- package/src/mcp_server.mjs +1100 -0
- package/src/rpc_client.mjs +3632 -0
- package/src/web_preview_server.mjs +420 -0
|
@@ -0,0 +1,1714 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Apify Pay-per-Event Execution, Native Platform Adapter & Crash-Safe Billing State Machine (src/apify_actor.mjs)
|
|
3
|
+
*
|
|
4
|
+
* Enforces deterministic Pay-per-Event billing and delivery invariants:
|
|
5
|
+
* 1. Explicit Accounting Separation: Tracks `extractedCount`, `deliveredCount`,
|
|
6
|
+
* `billableCount` (simulated unit-economics count), and confirmed `chargedEventCount`
|
|
7
|
+
* (`0` in local/MCP unbilled mode; strictly validated integer in Apify billing mode).
|
|
8
|
+
* 2. Native Zero-Dependency Apify Platform Adapter (`createApifyPlatformHooks` & `loadApifyActorInput`):
|
|
9
|
+
* Wires Apify Key-Value Store (`INPUT` & `ADPULSE_BILLING_CHECKPOINT`), Dataset (`pushData`),
|
|
10
|
+
* and Pay-per-Event Charging (`POST /v2/actor-runs/:runId/charge` with `Idempotency-Key`)
|
|
11
|
+
* when running on Apify (`APIFY_IS_AT_HOME=1` / `APIFY_TOKEN` + `APIFY_ACTOR_RUN_ID`) or local storage.
|
|
12
|
+
* 3. Crash-Safe KVS Checkpoint State Machine (`inFlightIds` -> `chargedIds` -> `deliveredIds`
|
|
13
|
+
* or `unknownOutcomeIds`), deterministic prefix delivery when `eventChargeLimitReached` returns
|
|
14
|
+
* partial `chargedCount`, cursor rewind on mid-stream halt/limit, and honest zero-event telemetry.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import crypto from 'node:crypto';
|
|
20
|
+
import {
|
|
21
|
+
resolveAdvertiserEntities,
|
|
22
|
+
fetchAdvertiserCreatives,
|
|
23
|
+
getCreativeDetailsById,
|
|
24
|
+
createEmptyUntrustedAdContent,
|
|
25
|
+
computeRewoundCursorForDeliveredPrefix,
|
|
26
|
+
verifyEgressUrlSsrfSafe,
|
|
27
|
+
CREATIVE_ID_RE,
|
|
28
|
+
ADVERTISER_ID_RE,
|
|
29
|
+
DEFAULT_RUN_DEADLINE_MS,
|
|
30
|
+
} from './rpc_client.mjs';
|
|
31
|
+
|
|
32
|
+
export const ACTOR_PRICING_CONTRACT = Object.freeze({
|
|
33
|
+
eventName: 'ad-creative-scraped',
|
|
34
|
+
actorStartFeeUsdc: 0.001,
|
|
35
|
+
priceUsdc: 0.0012,
|
|
36
|
+
costCeilingUsdc: 0.0001,
|
|
37
|
+
platformFeeRate: 0.2,
|
|
38
|
+
netPriceUsdc: 0.00096,
|
|
39
|
+
minMarginMultiplier: 2.5,
|
|
40
|
+
computeRatePerGbHourUsdc: 0.3,
|
|
41
|
+
maxMemoryMb: 256,
|
|
42
|
+
maxTimeoutSec: 25,
|
|
43
|
+
chunkSize: 25,
|
|
44
|
+
checkpointKey: 'ADPULSE_BILLING_CHECKPOINT',
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Computes content-addressed batch idempotency key (`sha256(runId + '\x1f' + sortedChunkIds)`).
|
|
49
|
+
*
|
|
50
|
+
* @param {string} runId
|
|
51
|
+
* @param {string[]} creativeIds
|
|
52
|
+
* @returns {string}
|
|
53
|
+
*/
|
|
54
|
+
export function computeChunkIdempotencyKey(runId, creativeIds) {
|
|
55
|
+
const safeRunId = String(runId || 'default_run');
|
|
56
|
+
const sortedIds = [...creativeIds].map((id) => String(id)).sort();
|
|
57
|
+
return crypto
|
|
58
|
+
.createHash('sha256')
|
|
59
|
+
.update(`${safeRunId}\x1f${sortedIds.join('\x1f')}`, 'utf8')
|
|
60
|
+
.digest('hex');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Computes unit-economics and margin telemetry for a single Actor/MCP execution run.
|
|
65
|
+
* Uses Apify's standard `$0.30 / GB-hour` compute rate (`0.25 GB` at `256MB`).
|
|
66
|
+
* When `billableCount === 0`, returns `economicsStatus: 'NO_BILLABLE_EVENTS'` with
|
|
67
|
+
* `withinCostCeiling: false` and `marginPositive: false`.
|
|
68
|
+
*
|
|
69
|
+
* @param {object} params
|
|
70
|
+
* @param {number} params.billableCount
|
|
71
|
+
* @param {number} [params.chargedEventCount=0]
|
|
72
|
+
* @param {number} params.durationMs
|
|
73
|
+
* @param {number} [params.memoryMb=256]
|
|
74
|
+
* @param {number} [params.requestCount=1]
|
|
75
|
+
* @returns {object}
|
|
76
|
+
*/
|
|
77
|
+
export function computeRunEconomicsTelemetry({
|
|
78
|
+
billableCount = 0,
|
|
79
|
+
chargedEventCount = 0,
|
|
80
|
+
durationMs = 100,
|
|
81
|
+
memoryMb =
|
|
82
|
+
Number(process.env.ACTOR_MEMORY_MBYTES || process.env.APIFY_MEMORY_MBYTES) ||
|
|
83
|
+
ACTOR_PRICING_CONTRACT.maxMemoryMb,
|
|
84
|
+
requestCount = 1,
|
|
85
|
+
} = {}) {
|
|
86
|
+
const safeCount = Math.max(0, Math.floor(Number(billableCount) || 0));
|
|
87
|
+
const safeChargedCount = Math.max(0, Math.floor(Number(chargedEventCount) || 0));
|
|
88
|
+
const safeDurationMs = Math.max(1, Number(durationMs) || 1);
|
|
89
|
+
const safeMemoryMb = Math.min(1024, Math.max(64, Number(memoryMb) || 256));
|
|
90
|
+
const safeRequests = Math.max(1, Math.floor(Number(requestCount) || 1));
|
|
91
|
+
|
|
92
|
+
// Apify compute rate: $0.30 per GB-hour; 256MB = 0.25 GB -> $0.075/hour -> $0.00002083/sec
|
|
93
|
+
const computeHours = safeDurationMs / 3600000;
|
|
94
|
+
const computeCostUsdc =
|
|
95
|
+
(safeMemoryMb / 1024) * computeHours * ACTOR_PRICING_CONTRACT.computeRatePerGbHourUsdc;
|
|
96
|
+
const networkOverheadUsdc = safeRequests * 0.0000002;
|
|
97
|
+
const totalRunCostUsdc = Number((computeCostUsdc + networkOverheadUsdc).toFixed(8));
|
|
98
|
+
const actorStartFeeUsdc = ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
|
|
99
|
+
|
|
100
|
+
const configuredFloorMultiplier = Number(
|
|
101
|
+
(ACTOR_PRICING_CONTRACT.netPriceUsdc / ACTOR_PRICING_CONTRACT.costCeilingUsdc).toFixed(2)
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
if (safeCount === 0) {
|
|
105
|
+
return {
|
|
106
|
+
economicsStatus: 'NO_BILLABLE_EVENTS',
|
|
107
|
+
billableCount: 0,
|
|
108
|
+
chargedEventCount: safeChargedCount,
|
|
109
|
+
durationMs: safeDurationMs,
|
|
110
|
+
memoryMb: safeMemoryMb,
|
|
111
|
+
requestCount: safeRequests,
|
|
112
|
+
actorStartFeeUsdc,
|
|
113
|
+
totalRunCostUsdc,
|
|
114
|
+
actualCostPerEventUsdc: 0,
|
|
115
|
+
costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
|
|
116
|
+
withinCostCeiling: false,
|
|
117
|
+
grossRevenueUsdc: 0,
|
|
118
|
+
netRevenueUsdc: 0,
|
|
119
|
+
configuredFloorMultiplier,
|
|
120
|
+
effectiveMarginMultiplier: 0,
|
|
121
|
+
marginPositive: false,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const actualCostPerEventUsdc = Number((totalRunCostUsdc / safeCount).toFixed(8));
|
|
126
|
+
const grossRevenueUsdc = Number((safeCount * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6));
|
|
127
|
+
const netRevenueUsdc = Number((safeCount * ACTOR_PRICING_CONTRACT.netPriceUsdc).toFixed(6));
|
|
128
|
+
const effectiveMarginMultiplier =
|
|
129
|
+
actualCostPerEventUsdc > 0
|
|
130
|
+
? Number((ACTOR_PRICING_CONTRACT.netPriceUsdc / actualCostPerEventUsdc).toFixed(2))
|
|
131
|
+
: configuredFloorMultiplier;
|
|
132
|
+
|
|
133
|
+
const withinCostCeiling = actualCostPerEventUsdc <= ACTOR_PRICING_CONTRACT.costCeilingUsdc;
|
|
134
|
+
const marginPositive =
|
|
135
|
+
withinCostCeiling &&
|
|
136
|
+
configuredFloorMultiplier >= ACTOR_PRICING_CONTRACT.minMarginMultiplier;
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
economicsStatus: marginPositive ? 'PROFITABLE' : 'COST_CEILING_EXCEEDED',
|
|
140
|
+
billableCount: safeCount,
|
|
141
|
+
chargedEventCount: safeChargedCount,
|
|
142
|
+
durationMs: safeDurationMs,
|
|
143
|
+
memoryMb: safeMemoryMb,
|
|
144
|
+
requestCount: safeRequests,
|
|
145
|
+
actorStartFeeUsdc,
|
|
146
|
+
totalRunCostUsdc,
|
|
147
|
+
actualCostPerEventUsdc,
|
|
148
|
+
costCeilingUsdc: ACTOR_PRICING_CONTRACT.costCeilingUsdc,
|
|
149
|
+
withinCostCeiling,
|
|
150
|
+
grossRevenueUsdc,
|
|
151
|
+
netRevenueUsdc,
|
|
152
|
+
configuredFloorMultiplier,
|
|
153
|
+
effectiveMarginMultiplier,
|
|
154
|
+
marginPositive,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Validates that an Apify API base URL is safe (`https://api.apify.com` by default).
|
|
160
|
+
* When `rawApiBaseUrl` is an internal HTTP endpoint on Apify Cloud and a valid HTTPS
|
|
161
|
+
* `fallbackPublicBaseUrl` (`APIFY_API_PUBLIC_BASE_URL`) is provided, uses the HTTPS public URL.
|
|
162
|
+
*
|
|
163
|
+
* @param {string} [rawApiBaseUrl]
|
|
164
|
+
* @param {string} [fallbackPublicBaseUrl]
|
|
165
|
+
* @returns {string}
|
|
166
|
+
*/
|
|
167
|
+
function resolveSafeApifyApiBaseUrl(rawApiBaseUrl, fallbackPublicBaseUrl = undefined) {
|
|
168
|
+
const candidate = String(rawApiBaseUrl || fallbackPublicBaseUrl || 'https://api.apify.com')
|
|
169
|
+
.trim()
|
|
170
|
+
.replace(/\/+$/, '');
|
|
171
|
+
const ssrf = verifyEgressUrlSsrfSafe(candidate);
|
|
172
|
+
if (!ssrf.safe) {
|
|
173
|
+
if (fallbackPublicBaseUrl) {
|
|
174
|
+
const publicCandidate = String(fallbackPublicBaseUrl).trim().replace(/\/+$/, '');
|
|
175
|
+
const publicSsrf = verifyEgressUrlSsrfSafe(publicCandidate);
|
|
176
|
+
if (publicSsrf.safe) {
|
|
177
|
+
return publicCandidate;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
throw new Error(`Invalid or unsafe APIFY_API_BASE_URL '${candidate}': ${ssrf.reason}`);
|
|
181
|
+
}
|
|
182
|
+
return candidate;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Loads Actor input JSON from CLI `--input-json`, `APIFY_INPUT_PATH`, local Apify storage
|
|
187
|
+
* (`$ACTOR_STORAGE_DIR` / `$APIFY_LOCAL_STORAGE_DIR/key_value_stores/default/INPUT.json`),
|
|
188
|
+
* or Apify Platform KVS REST API.
|
|
189
|
+
*
|
|
190
|
+
* @param {string[]} [cliArgs=[]]
|
|
191
|
+
* @param {object} [env=process.env]
|
|
192
|
+
* @param {Function} [fetchImpl=globalThis.fetch]
|
|
193
|
+
* @returns {Promise<object>}
|
|
194
|
+
*/
|
|
195
|
+
export async function loadApifyActorInput(
|
|
196
|
+
cliArgs = [],
|
|
197
|
+
env = process.env,
|
|
198
|
+
fetchImpl = globalThis.fetch
|
|
199
|
+
) {
|
|
200
|
+
const inputIdx = Array.isArray(cliArgs) ? cliArgs.indexOf('--input-json') : -1;
|
|
201
|
+
if (inputIdx !== -1 && cliArgs[inputIdx + 1]) {
|
|
202
|
+
const raw = fs.readFileSync(cliArgs[inputIdx + 1], 'utf8');
|
|
203
|
+
return JSON.parse(raw);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (env?.APIFY_INPUT_PATH && fs.existsSync(env.APIFY_INPUT_PATH)) {
|
|
207
|
+
const raw = fs.readFileSync(env.APIFY_INPUT_PATH, 'utf8');
|
|
208
|
+
return JSON.parse(raw);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const localStorageDir =
|
|
212
|
+
env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
|
|
213
|
+
if (localStorageDir) {
|
|
214
|
+
const kvsId =
|
|
215
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ||
|
|
216
|
+
env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID ||
|
|
217
|
+
'default';
|
|
218
|
+
const inputKey = env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT';
|
|
219
|
+
for (const candidateName of [`${inputKey}.json`, inputKey]) {
|
|
220
|
+
const localInputPath = path.join(
|
|
221
|
+
localStorageDir,
|
|
222
|
+
'key_value_stores',
|
|
223
|
+
kvsId,
|
|
224
|
+
candidateName
|
|
225
|
+
);
|
|
226
|
+
if (fs.existsSync(localInputPath)) {
|
|
227
|
+
return JSON.parse(fs.readFileSync(localInputPath, 'utf8'));
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const kvsId =
|
|
233
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
|
|
234
|
+
const token = env?.APIFY_TOKEN;
|
|
235
|
+
if (kvsId && token && typeof fetchImpl === 'function') {
|
|
236
|
+
const apiBase = resolveSafeApifyApiBaseUrl(
|
|
237
|
+
env?.APIFY_API_BASE_URL,
|
|
238
|
+
env?.APIFY_API_PUBLIC_BASE_URL
|
|
239
|
+
);
|
|
240
|
+
const inputKey = encodeURIComponent(
|
|
241
|
+
env?.ACTOR_INPUT_KEY || env?.APIFY_INPUT_KEY || 'INPUT'
|
|
242
|
+
);
|
|
243
|
+
const url = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${inputKey}`;
|
|
244
|
+
const controller = new AbortController();
|
|
245
|
+
const timer = setTimeout(() => controller.abort(), 4000);
|
|
246
|
+
try {
|
|
247
|
+
const res = await fetchImpl(url, {
|
|
248
|
+
method: 'GET',
|
|
249
|
+
headers: {
|
|
250
|
+
authorization: `Bearer ${token}`,
|
|
251
|
+
accept: 'application/json',
|
|
252
|
+
},
|
|
253
|
+
signal: controller.signal,
|
|
254
|
+
});
|
|
255
|
+
if (res.status === 200) {
|
|
256
|
+
const text = await res.text();
|
|
257
|
+
if (text && text.trim()) {
|
|
258
|
+
return JSON.parse(text);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
} finally {
|
|
262
|
+
clearTimeout(timer);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return {};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Creates native zero-dependency Apify Platform runtime hooks (`runId`, `isAtHome`,
|
|
271
|
+
* `loadCheckpoint`, `saveCheckpoint`, `listDeliveredIds`, `pushData`, `chargeEvent`, `saveOutput`)
|
|
272
|
+
* when Apify environment variables (`ACTOR_*` primary, `APIFY_*` fallback) or local
|
|
273
|
+
* storage directories are configured. Returns `{}` in pure local CLI/MCP mode.
|
|
274
|
+
*
|
|
275
|
+
* @param {object} [env=process.env]
|
|
276
|
+
* @param {Function} [fetchImpl=globalThis.fetch]
|
|
277
|
+
* @returns {object}
|
|
278
|
+
*/
|
|
279
|
+
export function createApifyPlatformHooks(env = process.env, fetchImpl = globalThis.fetch) {
|
|
280
|
+
const rawRunId = env?.ACTOR_RUN_ID || env?.APIFY_ACTOR_RUN_ID;
|
|
281
|
+
const runId = rawRunId ? String(rawRunId).trim() : undefined;
|
|
282
|
+
const token = env?.APIFY_TOKEN ? String(env.APIFY_TOKEN).trim() : undefined;
|
|
283
|
+
const rawDatasetId = env?.ACTOR_DEFAULT_DATASET_ID || env?.APIFY_DEFAULT_DATASET_ID;
|
|
284
|
+
const datasetId = rawDatasetId ? String(rawDatasetId).trim() : undefined;
|
|
285
|
+
const rawKvsId =
|
|
286
|
+
env?.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || env?.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
|
|
287
|
+
const kvsId = rawKvsId ? String(rawKvsId).trim() : undefined;
|
|
288
|
+
const localStorageDir =
|
|
289
|
+
env?.ACTOR_STORAGE_DIR || env?.APIFY_LOCAL_STORAGE_DIR || env?.CRAWLEE_STORAGE_DIR;
|
|
290
|
+
const isAtHome = env?.APIFY_IS_AT_HOME === '1' || env?.APIFY_IS_AT_HOME === 'true';
|
|
291
|
+
|
|
292
|
+
const hasCloudApi = Boolean((isAtHome || runId) && token && typeof fetchImpl === 'function');
|
|
293
|
+
if (!hasCloudApi && !localStorageDir) {
|
|
294
|
+
if (isAtHome) {
|
|
295
|
+
return { isAtHome: true, ...(runId ? { runId } : {}) };
|
|
296
|
+
}
|
|
297
|
+
return runId ? { runId } : {};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const hooks = {};
|
|
301
|
+
if (runId) {
|
|
302
|
+
hooks.runId = runId;
|
|
303
|
+
}
|
|
304
|
+
if (isAtHome || hasCloudApi) {
|
|
305
|
+
hooks.isAtHome = true;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
if (hasCloudApi) {
|
|
309
|
+
const apiBase = resolveSafeApifyApiBaseUrl(
|
|
310
|
+
env?.APIFY_API_BASE_URL,
|
|
311
|
+
env?.APIFY_API_PUBLIC_BASE_URL
|
|
312
|
+
);
|
|
313
|
+
const rawMaxChargeStr =
|
|
314
|
+
env?.ACTOR_MAX_TOTAL_CHARGE_USD !== undefined && env?.ACTOR_MAX_TOTAL_CHARGE_USD !== null
|
|
315
|
+
? String(env.ACTOR_MAX_TOTAL_CHARGE_USD).trim()
|
|
316
|
+
: '';
|
|
317
|
+
const maxTotalChargeUsdRaw = rawMaxChargeStr !== '' ? Number(rawMaxChargeStr) : NaN;
|
|
318
|
+
const hasMaxChargeUsd = Number.isFinite(maxTotalChargeUsdRaw) && maxTotalChargeUsdRaw >= 0;
|
|
319
|
+
const includeStartFeeFlag = String(env?.ACTOR_INCLUDE_START_FEE ?? '1')
|
|
320
|
+
.trim()
|
|
321
|
+
.toLowerCase();
|
|
322
|
+
const includeStartFee = includeStartFeeFlag !== '0' && includeStartFeeFlag !== 'false';
|
|
323
|
+
const explicitStartFeeUsd =
|
|
324
|
+
env?.ACTOR_START_FEE_USD !== undefined &&
|
|
325
|
+
env?.ACTOR_START_FEE_USD !== null &&
|
|
326
|
+
String(env.ACTOR_START_FEE_USD).trim() !== ''
|
|
327
|
+
? Number(env.ACTOR_START_FEE_USD)
|
|
328
|
+
: ACTOR_PRICING_CONTRACT.actorStartFeeUsdc;
|
|
329
|
+
const startFeeMicros =
|
|
330
|
+
includeStartFee && Number.isFinite(explicitStartFeeUsd) && explicitStartFeeUsd >= 0
|
|
331
|
+
? Math.round(explicitStartFeeUsd * 1_000_000)
|
|
332
|
+
: 0;
|
|
333
|
+
const unitPriceMicros = Math.round(ACTOR_PRICING_CONTRACT.priceUsdc * 1_000_000);
|
|
334
|
+
const maxBudgetMicros = hasMaxChargeUsd ? Math.round(maxTotalChargeUsdRaw * 1_000_000) : 0;
|
|
335
|
+
const maxPayableEvents = hasMaxChargeUsd
|
|
336
|
+
? Math.floor(Math.max(0, maxBudgetMicros - startFeeMicros) / unitPriceMicros)
|
|
337
|
+
: Infinity;
|
|
338
|
+
let cumulativeChargedInAdapter = 0;
|
|
339
|
+
let adapterChargeLimitLatched = false;
|
|
340
|
+
|
|
341
|
+
if (kvsId) {
|
|
342
|
+
const checkpointRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/${ACTOR_PRICING_CONTRACT.checkpointKey}`;
|
|
343
|
+
const outputRecordUrl = `${apiBase}/v2/key-value-stores/${encodeURIComponent(kvsId)}/records/OUTPUT`;
|
|
344
|
+
|
|
345
|
+
hooks.loadCheckpoint = async () => {
|
|
346
|
+
const controller = new AbortController();
|
|
347
|
+
const timer = setTimeout(() => controller.abort(), 250);
|
|
348
|
+
try {
|
|
349
|
+
const res = await fetchImpl(checkpointRecordUrl, {
|
|
350
|
+
method: 'GET',
|
|
351
|
+
headers: {
|
|
352
|
+
authorization: `Bearer ${token}`,
|
|
353
|
+
accept: 'application/json',
|
|
354
|
+
},
|
|
355
|
+
signal: controller.signal,
|
|
356
|
+
});
|
|
357
|
+
if (res.status === 404 || res.status === 204) {
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
360
|
+
if (res.status >= 200 && res.status < 300) {
|
|
361
|
+
const text = await res.text();
|
|
362
|
+
const loaded = text && text.trim() ? JSON.parse(text) : null;
|
|
363
|
+
if (loaded && Array.isArray(loaded.chargedIds)) {
|
|
364
|
+
cumulativeChargedInAdapter = loaded.chargedIds.length;
|
|
365
|
+
}
|
|
366
|
+
return loaded;
|
|
367
|
+
}
|
|
368
|
+
throw new Error(`Apify KVS loadCheckpoint failed with HTTP ${res.status}`);
|
|
369
|
+
} finally {
|
|
370
|
+
clearTimeout(timer);
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
hooks.saveCheckpoint = async (snapshot) => {
|
|
375
|
+
const controller = new AbortController();
|
|
376
|
+
const timer = setTimeout(() => controller.abort(), 250);
|
|
377
|
+
try {
|
|
378
|
+
const res = await fetchImpl(checkpointRecordUrl, {
|
|
379
|
+
method: 'PUT',
|
|
380
|
+
headers: {
|
|
381
|
+
authorization: `Bearer ${token}`,
|
|
382
|
+
'content-type': 'application/json; charset=utf-8',
|
|
383
|
+
},
|
|
384
|
+
body: JSON.stringify(snapshot),
|
|
385
|
+
signal: controller.signal,
|
|
386
|
+
});
|
|
387
|
+
if (res.status < 200 || res.status >= 300) {
|
|
388
|
+
throw new Error(`Apify KVS saveCheckpoint HTTP ${res.status}`);
|
|
389
|
+
}
|
|
390
|
+
} finally {
|
|
391
|
+
clearTimeout(timer);
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
hooks.saveOutput = async (outputRecord) => {
|
|
396
|
+
const controller = new AbortController();
|
|
397
|
+
const timer = setTimeout(() => controller.abort(), 250);
|
|
398
|
+
try {
|
|
399
|
+
const res = await fetchImpl(outputRecordUrl, {
|
|
400
|
+
method: 'PUT',
|
|
401
|
+
headers: {
|
|
402
|
+
authorization: `Bearer ${token}`,
|
|
403
|
+
'content-type': 'application/json; charset=utf-8',
|
|
404
|
+
},
|
|
405
|
+
body: JSON.stringify(outputRecord),
|
|
406
|
+
signal: controller.signal,
|
|
407
|
+
});
|
|
408
|
+
if (res.status < 200 || res.status >= 300) {
|
|
409
|
+
throw new Error(`Apify KVS saveOutput HTTP ${res.status}`);
|
|
410
|
+
}
|
|
411
|
+
} finally {
|
|
412
|
+
clearTimeout(timer);
|
|
413
|
+
}
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
if (datasetId) {
|
|
418
|
+
const datasetItemsUrl = `${apiBase}/v2/datasets/${encodeURIComponent(datasetId)}/items`;
|
|
419
|
+
hooks.listDeliveredIds = async () => {
|
|
420
|
+
const controller = new AbortController();
|
|
421
|
+
const timer = setTimeout(() => controller.abort(), 500);
|
|
422
|
+
try {
|
|
423
|
+
const res = await fetchImpl(`${datasetItemsUrl}?fields=creativeId&clean=1`, {
|
|
424
|
+
method: 'GET',
|
|
425
|
+
headers: {
|
|
426
|
+
authorization: `Bearer ${token}`,
|
|
427
|
+
accept: 'application/json',
|
|
428
|
+
},
|
|
429
|
+
signal: controller.signal,
|
|
430
|
+
});
|
|
431
|
+
if (res.status >= 200 && res.status < 300) {
|
|
432
|
+
const text = await res.text();
|
|
433
|
+
const parsed = text && text.trim() ? JSON.parse(text) : [];
|
|
434
|
+
const arr = Array.isArray(parsed)
|
|
435
|
+
? parsed
|
|
436
|
+
: Array.isArray(parsed?.data?.items)
|
|
437
|
+
? parsed.data.items
|
|
438
|
+
: [];
|
|
439
|
+
return arr
|
|
440
|
+
.map((row) => (typeof row?.creativeId === 'string' ? row.creativeId : null))
|
|
441
|
+
.filter(Boolean);
|
|
442
|
+
}
|
|
443
|
+
throw new Error(`Apify Dataset listDeliveredIds failed with HTTP ${res.status}`);
|
|
444
|
+
} finally {
|
|
445
|
+
clearTimeout(timer);
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
hooks.pushData = async (items) => {
|
|
450
|
+
const controller = new AbortController();
|
|
451
|
+
const timer = setTimeout(() => controller.abort(), 500);
|
|
452
|
+
try {
|
|
453
|
+
const res = await fetchImpl(datasetItemsUrl, {
|
|
454
|
+
method: 'POST',
|
|
455
|
+
headers: {
|
|
456
|
+
authorization: `Bearer ${token}`,
|
|
457
|
+
'content-type': 'application/json; charset=utf-8',
|
|
458
|
+
},
|
|
459
|
+
body: JSON.stringify(items),
|
|
460
|
+
signal: controller.signal,
|
|
461
|
+
});
|
|
462
|
+
if (res.status < 200 || res.status >= 300) {
|
|
463
|
+
throw new Error(`Apify Dataset pushData HTTP ${res.status}`);
|
|
464
|
+
}
|
|
465
|
+
} finally {
|
|
466
|
+
clearTimeout(timer);
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
if (runId) {
|
|
472
|
+
const chargeUrl = `${apiBase}/v2/actor-runs/${encodeURIComponent(runId)}/charge`;
|
|
473
|
+
hooks.chargeEvent = async (receipt) => {
|
|
474
|
+
const requestedCount = Math.max(0, Number(receipt?.count) || 0);
|
|
475
|
+
if (adapterChargeLimitLatched) {
|
|
476
|
+
return {
|
|
477
|
+
chargedCount: 0,
|
|
478
|
+
eventChargeLimitReached: true,
|
|
479
|
+
chargedCreativeIds: [],
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
const remainingAllowance = Number.isFinite(maxPayableEvents)
|
|
483
|
+
? Math.max(0, maxPayableEvents - cumulativeChargedInAdapter)
|
|
484
|
+
: requestedCount;
|
|
485
|
+
const countToCharge = Math.min(requestedCount, remainingAllowance);
|
|
486
|
+
const limitReached =
|
|
487
|
+
Number.isFinite(maxPayableEvents) &&
|
|
488
|
+
cumulativeChargedInAdapter + countToCharge >= maxPayableEvents &&
|
|
489
|
+
countToCharge < requestedCount;
|
|
490
|
+
|
|
491
|
+
if (countToCharge === 0) {
|
|
492
|
+
return {
|
|
493
|
+
chargedCount: 0,
|
|
494
|
+
eventChargeLimitReached: Number.isFinite(maxPayableEvents),
|
|
495
|
+
chargedCreativeIds: [],
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
const controller = new AbortController();
|
|
500
|
+
const timer = setTimeout(() => controller.abort(), 600);
|
|
501
|
+
try {
|
|
502
|
+
const res = await fetchImpl(chargeUrl, {
|
|
503
|
+
method: 'POST',
|
|
504
|
+
headers: {
|
|
505
|
+
authorization: `Bearer ${token}`,
|
|
506
|
+
'content-type': 'application/json; charset=utf-8',
|
|
507
|
+
'idempotency-key': receipt.idempotencyKey,
|
|
508
|
+
},
|
|
509
|
+
body: JSON.stringify({
|
|
510
|
+
eventName: receipt.eventName,
|
|
511
|
+
count: countToCharge,
|
|
512
|
+
}),
|
|
513
|
+
signal: controller.signal,
|
|
514
|
+
});
|
|
515
|
+
if (res.status === 402) {
|
|
516
|
+
// Platform enforced spending ceiling (HTTP 402 Payment Required): latch so no further charge POSTs occur
|
|
517
|
+
adapterChargeLimitLatched = true;
|
|
518
|
+
return {
|
|
519
|
+
chargedCount: 0,
|
|
520
|
+
eventChargeLimitReached: true,
|
|
521
|
+
chargedCreativeIds: [],
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
if (res.status < 200 || res.status >= 300) {
|
|
525
|
+
throw new Error(`Apify Pay-per-Event charge HTTP ${res.status}`);
|
|
526
|
+
}
|
|
527
|
+
const rawText = await res.text();
|
|
528
|
+
const parsed = rawText && rawText.trim() ? JSON.parse(rawText) : {};
|
|
529
|
+
const dataObj = parsed?.data && typeof parsed.data === 'object' ? parsed.data : parsed;
|
|
530
|
+
// Apify REST `POST /v2/actor-runs/{runId}/charge` returns HTTP 201 Created with `{}` on success.
|
|
531
|
+
// Only fallback to `countToCharge` when `chargedCount` is genuinely omitted; if present but invalid, throw fail-closed.
|
|
532
|
+
const hasExplicitChargedCount =
|
|
533
|
+
Boolean(dataObj) &&
|
|
534
|
+
typeof dataObj === 'object' &&
|
|
535
|
+
Object.prototype.hasOwnProperty.call(dataObj, 'chargedCount') &&
|
|
536
|
+
dataObj.chargedCount !== undefined;
|
|
537
|
+
if (
|
|
538
|
+
hasExplicitChargedCount &&
|
|
539
|
+
(!Number.isInteger(dataObj.chargedCount) ||
|
|
540
|
+
dataObj.chargedCount < 0 ||
|
|
541
|
+
dataObj.chargedCount > countToCharge)
|
|
542
|
+
) {
|
|
543
|
+
throw new Error(
|
|
544
|
+
`Apify Pay-per-Event charge response contained invalid chargedCount: ${String(dataObj.chargedCount)}`
|
|
545
|
+
);
|
|
546
|
+
}
|
|
547
|
+
const resolvedChargedCount = hasExplicitChargedCount
|
|
548
|
+
? dataObj.chargedCount
|
|
549
|
+
: countToCharge;
|
|
550
|
+
cumulativeChargedInAdapter += resolvedChargedCount;
|
|
551
|
+
const resolvedLimitReached =
|
|
552
|
+
Boolean(dataObj?.eventChargeLimitReached) ||
|
|
553
|
+
limitReached ||
|
|
554
|
+
(Number.isFinite(maxPayableEvents) && cumulativeChargedInAdapter >= maxPayableEvents);
|
|
555
|
+
const subsetIds = Array.isArray(dataObj?.chargedCreativeIds)
|
|
556
|
+
? dataObj.chargedCreativeIds
|
|
557
|
+
: Array.isArray(receipt?.creativeIds)
|
|
558
|
+
? receipt.creativeIds.slice(0, resolvedChargedCount)
|
|
559
|
+
: undefined;
|
|
560
|
+
return {
|
|
561
|
+
chargedCount: resolvedChargedCount,
|
|
562
|
+
eventChargeLimitReached: resolvedLimitReached,
|
|
563
|
+
...(subsetIds ? { chargedCreativeIds: subsetIds } : {}),
|
|
564
|
+
};
|
|
565
|
+
} finally {
|
|
566
|
+
clearTimeout(timer);
|
|
567
|
+
}
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
return hooks;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// Local Apify storage directory mode (`APIFY_LOCAL_STORAGE_DIR`)
|
|
575
|
+
const storeName = kvsId || 'default';
|
|
576
|
+
const dsName = datasetId || 'default';
|
|
577
|
+
const kvsDir = path.join(localStorageDir, 'key_value_stores', storeName);
|
|
578
|
+
const dsDir = path.join(localStorageDir, 'datasets', dsName);
|
|
579
|
+
const checkpointPath = path.join(kvsDir, `${ACTOR_PRICING_CONTRACT.checkpointKey}.json`);
|
|
580
|
+
const outputPath = path.join(kvsDir, 'OUTPUT.json');
|
|
581
|
+
|
|
582
|
+
hooks.loadCheckpoint = async () => {
|
|
583
|
+
if (!fs.existsSync(checkpointPath)) return null;
|
|
584
|
+
return JSON.parse(fs.readFileSync(checkpointPath, 'utf8'));
|
|
585
|
+
};
|
|
586
|
+
|
|
587
|
+
hooks.saveCheckpoint = async (snapshot) => {
|
|
588
|
+
fs.mkdirSync(kvsDir, { recursive: true });
|
|
589
|
+
fs.writeFileSync(checkpointPath, JSON.stringify(snapshot, null, 2), 'utf8');
|
|
590
|
+
};
|
|
591
|
+
|
|
592
|
+
hooks.saveOutput = async (outputRecord) => {
|
|
593
|
+
fs.mkdirSync(kvsDir, { recursive: true });
|
|
594
|
+
fs.writeFileSync(outputPath, JSON.stringify(outputRecord, null, 2), 'utf8');
|
|
595
|
+
};
|
|
596
|
+
|
|
597
|
+
hooks.listDeliveredIds = async () => {
|
|
598
|
+
if (!fs.existsSync(dsDir)) return [];
|
|
599
|
+
const existingFiles = fs.readdirSync(dsDir).filter((f) => f.endsWith('.json'));
|
|
600
|
+
const ids = [];
|
|
601
|
+
for (const file of existingFiles) {
|
|
602
|
+
try {
|
|
603
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(dsDir, file), 'utf8'));
|
|
604
|
+
if (typeof parsed?.creativeId === 'string') {
|
|
605
|
+
ids.push(parsed.creativeId);
|
|
606
|
+
}
|
|
607
|
+
} catch {
|
|
608
|
+
// Ignore unreadable partial files
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
return ids;
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
hooks.pushData = async (items) => {
|
|
615
|
+
fs.mkdirSync(dsDir, { recursive: true });
|
|
616
|
+
const existingSeqs = fs
|
|
617
|
+
.readdirSync(dsDir)
|
|
618
|
+
.map((f) => {
|
|
619
|
+
const m = /^(\d+)\.json$/.exec(f);
|
|
620
|
+
return m ? Number(m[1]) : 0;
|
|
621
|
+
})
|
|
622
|
+
.filter((n) => Number.isInteger(n) && n > 0);
|
|
623
|
+
let nextSeq = existingSeqs.length > 0 ? Math.max(...existingSeqs) + 1 : 1;
|
|
624
|
+
const list = Array.isArray(items) ? items : [items];
|
|
625
|
+
for (const item of list) {
|
|
626
|
+
const fileName = `${String(nextSeq++).padStart(9, '0')}.json`;
|
|
627
|
+
fs.writeFileSync(path.join(dsDir, fileName), JSON.stringify(item, null, 2), 'utf8');
|
|
628
|
+
}
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
return hooks;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Normalizes a persisted KVS billing checkpoint object.
|
|
636
|
+
* If the checkpoint belongs to a different `runId`, discards stale state and starts fresh.
|
|
637
|
+
*
|
|
638
|
+
* @param {unknown} rawState
|
|
639
|
+
* @param {string} runId
|
|
640
|
+
* @returns {object}
|
|
641
|
+
*/
|
|
642
|
+
function normalizeBillingCheckpointState(rawState, runId) {
|
|
643
|
+
const s = rawState && typeof rawState === 'object' && !Array.isArray(rawState) ? rawState : {};
|
|
644
|
+
if (s.runId && String(s.runId).trim() !== String(runId).trim()) {
|
|
645
|
+
return {
|
|
646
|
+
runId: String(runId || 'default_run'),
|
|
647
|
+
chargedIds: [],
|
|
648
|
+
deliveredIds: [],
|
|
649
|
+
inFlightIds: [],
|
|
650
|
+
unknownOutcomeIds: [],
|
|
651
|
+
stagedItemsById: {},
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
const toUniqueIdArray = (arr) =>
|
|
656
|
+
Array.isArray(arr)
|
|
657
|
+
? Array.from(new Set(arr.map((x) => String(x || '').trim()).filter((x) => CREATIVE_ID_RE.test(x))))
|
|
658
|
+
: [];
|
|
659
|
+
|
|
660
|
+
const stagedItemsById = {};
|
|
661
|
+
if (s.stagedItemsById && typeof s.stagedItemsById === 'object' && !Array.isArray(s.stagedItemsById)) {
|
|
662
|
+
for (const [k, v] of Object.entries(s.stagedItemsById)) {
|
|
663
|
+
if (CREATIVE_ID_RE.test(k) && v && typeof v === 'object') {
|
|
664
|
+
stagedItemsById[k] = v;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
return {
|
|
670
|
+
runId: String(s.runId || runId || 'default_run'),
|
|
671
|
+
chargedIds: toUniqueIdArray(s.chargedIds),
|
|
672
|
+
deliveredIds: toUniqueIdArray(s.deliveredIds),
|
|
673
|
+
inFlightIds: toUniqueIdArray(s.inFlightIds),
|
|
674
|
+
unknownOutcomeIds: toUniqueIdArray(s.unknownOutcomeIds),
|
|
675
|
+
stagedItemsById,
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Executes the end-to-end Google Ads Transparency Actor workflow with crash-safe billing
|
|
681
|
+
* checkpointing, strict `ChargeResult` validation, deterministic partial-limit prefix delivery,
|
|
682
|
+
* mid-stream cursor rewind, and fail-closed zero-charge protection.
|
|
683
|
+
*
|
|
684
|
+
* @param {object} input
|
|
685
|
+
* @param {object} [hooks]
|
|
686
|
+
* @param {Function} [hooks.pushData]
|
|
687
|
+
* @param {Function} [hooks.chargeEvent]
|
|
688
|
+
* @param {Function} [hooks.loadCheckpoint]
|
|
689
|
+
* @param {Function} [hooks.saveCheckpoint]
|
|
690
|
+
* @param {Function} [hooks.fetchImpl]
|
|
691
|
+
* @param {Function} [hooks.nowMs]
|
|
692
|
+
* @param {string} [hooks.runId]
|
|
693
|
+
* @returns {Promise<object>}
|
|
694
|
+
*/
|
|
695
|
+
export async function runGoogleAdsTransparencyActor(input = {}, hooks = {}) {
|
|
696
|
+
const nowMs = typeof hooks.nowMs === 'function' ? hooks.nowMs : () => Date.now();
|
|
697
|
+
const startedAtMs = nowMs();
|
|
698
|
+
// Authoritative runId precedence: hooks.runId -> ACTOR_RUN_ID -> APIFY_ACTOR_RUN_ID -> input.runId -> 'default_run'
|
|
699
|
+
const runId = String(
|
|
700
|
+
hooks.runId ||
|
|
701
|
+
process.env.ACTOR_RUN_ID ||
|
|
702
|
+
process.env.APIFY_ACTOR_RUN_ID ||
|
|
703
|
+
input?.runId ||
|
|
704
|
+
'default_run'
|
|
705
|
+
);
|
|
706
|
+
const hasChargeHook = typeof hooks.chargeEvent === 'function';
|
|
707
|
+
const billingMode =
|
|
708
|
+
hasChargeHook || hooks.isAtHome === true ? 'APIFY_PAY_PER_EVENT' : 'LOCAL_UNBILLED';
|
|
709
|
+
|
|
710
|
+
const maxRunDurationMs = Math.min(
|
|
711
|
+
DEFAULT_RUN_DEADLINE_MS,
|
|
712
|
+
Math.max(1000, Number(input?.maxRunDurationMs) || DEFAULT_RUN_DEADLINE_MS)
|
|
713
|
+
);
|
|
714
|
+
const deadlineAtMs = startedAtMs + maxRunDurationMs;
|
|
715
|
+
const billingReserveMs =
|
|
716
|
+
maxRunDurationMs >= 6000 ? 2500 : Math.min(1000, Math.floor(maxRunDurationMs * 0.25));
|
|
717
|
+
const rpcDeadlineAtMs = hasChargeHook
|
|
718
|
+
? Math.max(startedAtMs + 500, deadlineAtMs - billingReserveMs)
|
|
719
|
+
: deadlineAtMs;
|
|
720
|
+
const step4ReserveMs = hasChargeHook
|
|
721
|
+
? Math.min(2100, Math.floor(billingReserveMs * 0.85))
|
|
722
|
+
: 0;
|
|
723
|
+
|
|
724
|
+
const finalizeActorReturn = async (finalResult) => {
|
|
725
|
+
if (typeof hooks.saveOutput === 'function' && nowMs() < deadlineAtMs) {
|
|
726
|
+
try {
|
|
727
|
+
await hooks.saveOutput({
|
|
728
|
+
ok: finalResult.ok,
|
|
729
|
+
status: finalResult.status,
|
|
730
|
+
reason: finalResult.reason || null,
|
|
731
|
+
billingMode: finalResult.billingMode,
|
|
732
|
+
extractedCount: finalResult.extractedCount,
|
|
733
|
+
deliveredCount: finalResult.deliveredCount,
|
|
734
|
+
billableCount: finalResult.billableCount,
|
|
735
|
+
chargedEventCount: finalResult.chargedEventCount,
|
|
736
|
+
pagination: finalResult.pagination,
|
|
737
|
+
...(Array.isArray(finalResult.warnings) && finalResult.warnings.length > 0
|
|
738
|
+
? { warnings: finalResult.warnings }
|
|
739
|
+
: {}),
|
|
740
|
+
queryMeta: finalResult.queryMeta,
|
|
741
|
+
selectedEntities: finalResult.selectedEntities,
|
|
742
|
+
telemetry: finalResult.telemetry,
|
|
743
|
+
});
|
|
744
|
+
} catch {
|
|
745
|
+
// Non-fatal KVS summary write
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
return finalResult;
|
|
749
|
+
};
|
|
750
|
+
|
|
751
|
+
const pushDataWithRetry = async (itemsToPush) => {
|
|
752
|
+
if (
|
|
753
|
+
typeof hooks.pushData !== 'function' ||
|
|
754
|
+
!Array.isArray(itemsToPush) ||
|
|
755
|
+
itemsToPush.length === 0
|
|
756
|
+
) {
|
|
757
|
+
return { ok: true };
|
|
758
|
+
}
|
|
759
|
+
let pendingItems = [...itemsToPush];
|
|
760
|
+
let lastErr = null;
|
|
761
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
762
|
+
try {
|
|
763
|
+
await hooks.pushData(pendingItems);
|
|
764
|
+
return { ok: true };
|
|
765
|
+
} catch (err) {
|
|
766
|
+
lastErr = err;
|
|
767
|
+
if (attempt < 2) {
|
|
768
|
+
if (nowMs() + 1500 >= deadlineAtMs || typeof hooks.listDeliveredIds !== 'function') {
|
|
769
|
+
return { ok: false, error: lastErr };
|
|
770
|
+
}
|
|
771
|
+
try {
|
|
772
|
+
const committedIds = new Set(await hooks.listDeliveredIds());
|
|
773
|
+
pendingItems = pendingItems.filter((it) => !committedIds.has(it.creativeId));
|
|
774
|
+
if (pendingItems.length === 0) {
|
|
775
|
+
return { ok: true };
|
|
776
|
+
}
|
|
777
|
+
} catch (reconcileErr) {
|
|
778
|
+
return { ok: false, error: reconcileErr || lastErr };
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
return { ok: false, error: lastErr };
|
|
784
|
+
};
|
|
785
|
+
|
|
786
|
+
// Step 0.0: Fail-Closed At-Home Cloud Hook Completeness Gate
|
|
787
|
+
// In Apify Cloud (`hooks.isAtHome === true`), require all 4 platform hooks (`chargeEvent`,
|
|
788
|
+
// `pushData`, `loadCheckpoint`, `saveCheckpoint`) before any RPC fetch or billing occurs.
|
|
789
|
+
if (
|
|
790
|
+
hooks.isAtHome === true &&
|
|
791
|
+
(typeof hooks.chargeEvent !== 'function' ||
|
|
792
|
+
typeof hooks.pushData !== 'function' ||
|
|
793
|
+
typeof hooks.loadCheckpoint !== 'function' ||
|
|
794
|
+
typeof hooks.saveCheckpoint !== 'function')
|
|
795
|
+
) {
|
|
796
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
797
|
+
return finalizeActorReturn({
|
|
798
|
+
ok: false,
|
|
799
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
800
|
+
reason:
|
|
801
|
+
'Incomplete Apify Cloud runtime hooks (missing chargeEvent, pushData, loadCheckpoint, or saveCheckpoint in at-home mode)',
|
|
802
|
+
billingMode,
|
|
803
|
+
queryMeta: null,
|
|
804
|
+
selectedEntities: [],
|
|
805
|
+
items: [],
|
|
806
|
+
extractedCount: 0,
|
|
807
|
+
deliveredCount: 0,
|
|
808
|
+
billableCount: 0,
|
|
809
|
+
chargedEventCount: 0,
|
|
810
|
+
unknownOutcomeIds: [],
|
|
811
|
+
chargeReceipts: [],
|
|
812
|
+
pagination: {
|
|
813
|
+
nextPageToken: input?.pageToken || null,
|
|
814
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
815
|
+
paginationIncompleteReason: 'CLOUD_HOOKS_INCOMPLETE',
|
|
816
|
+
},
|
|
817
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
818
|
+
billableCount: 0,
|
|
819
|
+
chargedEventCount: 0,
|
|
820
|
+
durationMs,
|
|
821
|
+
requestCount: 1,
|
|
822
|
+
}),
|
|
823
|
+
});
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// Step 0: Load & Reconcile Crash-Safe Billing Checkpoint (fail-closed if loadCheckpoint throws)
|
|
827
|
+
let rawLoadedCheckpoint = null;
|
|
828
|
+
if (typeof hooks.loadCheckpoint === 'function') {
|
|
829
|
+
try {
|
|
830
|
+
rawLoadedCheckpoint = await hooks.loadCheckpoint();
|
|
831
|
+
} catch (cpErr) {
|
|
832
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
833
|
+
return finalizeActorReturn({
|
|
834
|
+
ok: false,
|
|
835
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
836
|
+
reason: `Failed to load billing checkpoint fail-closed: ${cpErr?.message || String(cpErr)}`,
|
|
837
|
+
billingMode,
|
|
838
|
+
queryMeta: null,
|
|
839
|
+
selectedEntities: [],
|
|
840
|
+
items: [],
|
|
841
|
+
extractedCount: 0,
|
|
842
|
+
deliveredCount: 0,
|
|
843
|
+
billableCount: 0,
|
|
844
|
+
chargedEventCount: 0,
|
|
845
|
+
unknownOutcomeIds: [],
|
|
846
|
+
chargeReceipts: [],
|
|
847
|
+
pagination: {
|
|
848
|
+
nextPageToken: input?.pageToken || null,
|
|
849
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
850
|
+
paginationIncompleteReason: 'CHECKPOINT_LOAD_FAILED',
|
|
851
|
+
},
|
|
852
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
853
|
+
billableCount: 0,
|
|
854
|
+
chargedEventCount: 0,
|
|
855
|
+
durationMs,
|
|
856
|
+
requestCount: 1,
|
|
857
|
+
}),
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
const checkpoint = normalizeBillingCheckpointState(rawLoadedCheckpoint, runId);
|
|
862
|
+
|
|
863
|
+
const chargedSet = new Set(checkpoint.chargedIds);
|
|
864
|
+
const deliveredSet = new Set(checkpoint.deliveredIds);
|
|
865
|
+
const unknownOutcomeSet = new Set(checkpoint.unknownOutcomeIds);
|
|
866
|
+
let inFlightSet = new Set(checkpoint.inFlightIds);
|
|
867
|
+
const stagedItemsById = { ...checkpoint.stagedItemsById };
|
|
868
|
+
|
|
869
|
+
const persistCheckpoint = async () => {
|
|
870
|
+
const snapshot = {
|
|
871
|
+
runId,
|
|
872
|
+
chargedIds: Array.from(chargedSet),
|
|
873
|
+
deliveredIds: Array.from(deliveredSet),
|
|
874
|
+
inFlightIds: Array.from(inFlightSet),
|
|
875
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
876
|
+
stagedItemsById,
|
|
877
|
+
};
|
|
878
|
+
if (typeof hooks.saveCheckpoint === 'function') {
|
|
879
|
+
await hooks.saveCheckpoint(snapshot);
|
|
880
|
+
}
|
|
881
|
+
return snapshot;
|
|
882
|
+
};
|
|
883
|
+
|
|
884
|
+
const reconciledItems = [];
|
|
885
|
+
let deliveredCount = deliveredSet.size;
|
|
886
|
+
|
|
887
|
+
// Step 0a: Reconcile any already-paid items (`chargedIds \ deliveredIds`) without re-charging.
|
|
888
|
+
// First check `hooks.listDeliveredIds()` if available so items already committed to dataset before a lost ACK
|
|
889
|
+
// are marked delivered without duplicate dataset POSTs. Fail closed if dataset listing throws.
|
|
890
|
+
const paidUndeliveredIds = Array.from(chargedSet).filter((id) => !deliveredSet.has(id));
|
|
891
|
+
if (paidUndeliveredIds.length > 0) {
|
|
892
|
+
let alreadyInDatasetSet = new Set();
|
|
893
|
+
if (typeof hooks.listDeliveredIds === 'function') {
|
|
894
|
+
try {
|
|
895
|
+
const existingDsIds = await hooks.listDeliveredIds();
|
|
896
|
+
if (Array.isArray(existingDsIds)) {
|
|
897
|
+
alreadyInDatasetSet = new Set(existingDsIds);
|
|
898
|
+
}
|
|
899
|
+
} catch (dsListErr) {
|
|
900
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
901
|
+
return finalizeActorReturn({
|
|
902
|
+
ok: false,
|
|
903
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
904
|
+
reason: `Failed to verify delivered dataset IDs during reconciliation: ${dsListErr?.message || String(dsListErr)}`,
|
|
905
|
+
billingMode,
|
|
906
|
+
queryMeta: null,
|
|
907
|
+
selectedEntities: [],
|
|
908
|
+
items: [],
|
|
909
|
+
extractedCount: 0,
|
|
910
|
+
deliveredCount,
|
|
911
|
+
billableCount: 0,
|
|
912
|
+
chargedEventCount: chargedSet.size,
|
|
913
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
914
|
+
chargeReceipts: [],
|
|
915
|
+
pagination: {
|
|
916
|
+
nextPageToken: input?.pageToken || null,
|
|
917
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
918
|
+
paginationIncompleteReason: 'DATASET_RECONCILIATION_FAILED',
|
|
919
|
+
},
|
|
920
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
921
|
+
billableCount: 0,
|
|
922
|
+
chargedEventCount: chargedSet.size,
|
|
923
|
+
durationMs,
|
|
924
|
+
requestCount: 1,
|
|
925
|
+
}),
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
const deliverablePairs = [];
|
|
931
|
+
for (const id of paidUndeliveredIds) {
|
|
932
|
+
const staged = stagedItemsById[id];
|
|
933
|
+
if (alreadyInDatasetSet.has(id)) {
|
|
934
|
+
deliveredSet.add(id);
|
|
935
|
+
if (staged && typeof staged === 'object' && staged.creativeId === id) {
|
|
936
|
+
reconciledItems.push(staged);
|
|
937
|
+
}
|
|
938
|
+
} else if (staged && typeof staged === 'object' && staged.creativeId === id) {
|
|
939
|
+
deliverablePairs.push({ id, item: staged });
|
|
940
|
+
} else {
|
|
941
|
+
unknownOutcomeSet.add(id);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
if (deliverablePairs.length > 0) {
|
|
946
|
+
const itemsToDeliver = deliverablePairs.map((p) => p.item);
|
|
947
|
+
const pushRes = await pushDataWithRetry(itemsToDeliver);
|
|
948
|
+
if (!pushRes.ok) {
|
|
949
|
+
await persistCheckpoint();
|
|
950
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
951
|
+
return finalizeActorReturn({
|
|
952
|
+
ok: false,
|
|
953
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
954
|
+
reason: `Dataset pushData failed during reconciliation: ${pushRes.error?.message || String(pushRes.error)}`,
|
|
955
|
+
billingMode,
|
|
956
|
+
queryMeta: null,
|
|
957
|
+
selectedEntities: [],
|
|
958
|
+
items: reconciledItems,
|
|
959
|
+
extractedCount: 0,
|
|
960
|
+
deliveredCount,
|
|
961
|
+
billableCount: reconciledItems.length,
|
|
962
|
+
chargedEventCount: chargedSet.size,
|
|
963
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
964
|
+
chargeReceipts: [],
|
|
965
|
+
pagination: {
|
|
966
|
+
nextPageToken: input?.pageToken || null,
|
|
967
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
968
|
+
paginationIncompleteReason: 'DATASET_PUSH_FAILED',
|
|
969
|
+
},
|
|
970
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
971
|
+
billableCount: reconciledItems.length,
|
|
972
|
+
chargedEventCount: chargedSet.size,
|
|
973
|
+
durationMs,
|
|
974
|
+
requestCount: 1,
|
|
975
|
+
}),
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
for (const { id, item } of deliverablePairs) {
|
|
979
|
+
deliveredSet.add(id);
|
|
980
|
+
reconciledItems.push(item);
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
deliveredCount = deliveredSet.size;
|
|
984
|
+
await persistCheckpoint();
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// Step 0b: Any leftover `inFlightIds` from a prior interrupted run transition to `unknownOutcomeIds`
|
|
988
|
+
if (inFlightSet.size > 0) {
|
|
989
|
+
for (const id of inFlightSet) {
|
|
990
|
+
unknownOutcomeSet.add(id);
|
|
991
|
+
}
|
|
992
|
+
inFlightSet = new Set();
|
|
993
|
+
await persistCheckpoint();
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
if (unknownOutcomeSet.size > 0 && input?.strictBillingResume !== false) {
|
|
997
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
998
|
+
return finalizeActorReturn({
|
|
999
|
+
ok: false,
|
|
1000
|
+
status: 'PARTIAL_BILLING_HALT',
|
|
1001
|
+
reason: `Halted due to ${unknownOutcomeSet.size} unreconciled creative(s) in UNKNOWN_CHARGE_OUTCOME state`,
|
|
1002
|
+
billingMode,
|
|
1003
|
+
queryMeta: null,
|
|
1004
|
+
selectedEntities: [],
|
|
1005
|
+
items: reconciledItems,
|
|
1006
|
+
extractedCount: 0,
|
|
1007
|
+
deliveredCount,
|
|
1008
|
+
billableCount: reconciledItems.length,
|
|
1009
|
+
chargedEventCount: chargedSet.size,
|
|
1010
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1011
|
+
chargeReceipts: [],
|
|
1012
|
+
pagination: {
|
|
1013
|
+
nextPageToken: input?.pageToken || null,
|
|
1014
|
+
paginationStatus: 'PARTIAL_BILLING_HALT',
|
|
1015
|
+
paginationIncompleteReason: 'UNKNOWN_CHARGE_OUTCOME',
|
|
1016
|
+
},
|
|
1017
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1018
|
+
billableCount: reconciledItems.length,
|
|
1019
|
+
chargedEventCount: chargedSet.size,
|
|
1020
|
+
durationMs,
|
|
1021
|
+
requestCount: 1,
|
|
1022
|
+
}),
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
const sleepFn =
|
|
1027
|
+
typeof hooks.sleepFn === 'function'
|
|
1028
|
+
? hooks.sleepFn
|
|
1029
|
+
: typeof input?.sleepFn === 'function'
|
|
1030
|
+
? input.sleepFn
|
|
1031
|
+
: undefined;
|
|
1032
|
+
const randomFn =
|
|
1033
|
+
typeof hooks.randomFn === 'function'
|
|
1034
|
+
? hooks.randomFn
|
|
1035
|
+
: typeof input?.randomFn === 'function'
|
|
1036
|
+
? input.randomFn
|
|
1037
|
+
: undefined;
|
|
1038
|
+
|
|
1039
|
+
// Step 1: Resolve Advertiser Entities
|
|
1040
|
+
const entityRes = await resolveAdvertiserEntities({
|
|
1041
|
+
query: input?.query,
|
|
1042
|
+
advertiserId: input?.advertiserId,
|
|
1043
|
+
advertiserName: input?.advertiserName,
|
|
1044
|
+
countryCode: input?.countryCode,
|
|
1045
|
+
includeSubsidiaries: input?.includeSubsidiaries,
|
|
1046
|
+
maxEntities: input?.maxEntities,
|
|
1047
|
+
fetchImpl: hooks.fetchImpl,
|
|
1048
|
+
nowFn: nowMs,
|
|
1049
|
+
sleepFn,
|
|
1050
|
+
randomFn,
|
|
1051
|
+
timeoutMs: input?.timeoutMs,
|
|
1052
|
+
maxRetries: input?.maxRetries,
|
|
1053
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1054
|
+
});
|
|
1055
|
+
|
|
1056
|
+
if (!entityRes.ok) {
|
|
1057
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1058
|
+
const isPreChargeDeadline =
|
|
1059
|
+
hasChargeHook &&
|
|
1060
|
+
nowMs() >= rpcDeadlineAtMs &&
|
|
1061
|
+
/deadline exceeded/i.test(String(entityRes.reason || ''));
|
|
1062
|
+
return finalizeActorReturn({
|
|
1063
|
+
ok: false,
|
|
1064
|
+
status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : entityRes.status,
|
|
1065
|
+
reason: entityRes.reason,
|
|
1066
|
+
billingMode,
|
|
1067
|
+
queryMeta: entityRes.queryMeta,
|
|
1068
|
+
selectedEntities: [],
|
|
1069
|
+
items: reconciledItems,
|
|
1070
|
+
extractedCount: 0,
|
|
1071
|
+
deliveredCount,
|
|
1072
|
+
billableCount: reconciledItems.length,
|
|
1073
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1074
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1075
|
+
chargeReceipts: [],
|
|
1076
|
+
pagination: {
|
|
1077
|
+
nextPageToken: input?.pageToken || null,
|
|
1078
|
+
paginationStatus: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_HAS_MORE' : entityRes.status,
|
|
1079
|
+
paginationIncompleteReason: isPreChargeDeadline
|
|
1080
|
+
? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
1081
|
+
: entityRes.reason || null,
|
|
1082
|
+
},
|
|
1083
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1084
|
+
billableCount: reconciledItems.length,
|
|
1085
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1086
|
+
durationMs,
|
|
1087
|
+
requestCount: Math.max(1, entityRes.rpcAttempts),
|
|
1088
|
+
}),
|
|
1089
|
+
});
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
if (entityRes.status === 'EMPTY_RESULT' || entityRes.selectedEntities.length === 0) {
|
|
1093
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1094
|
+
return finalizeActorReturn({
|
|
1095
|
+
ok: true,
|
|
1096
|
+
status: 'EMPTY_RESULT',
|
|
1097
|
+
billingMode,
|
|
1098
|
+
queryMeta: entityRes.queryMeta,
|
|
1099
|
+
selectedEntities: [],
|
|
1100
|
+
items: reconciledItems,
|
|
1101
|
+
extractedCount: 0,
|
|
1102
|
+
deliveredCount,
|
|
1103
|
+
billableCount: reconciledItems.length,
|
|
1104
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1105
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1106
|
+
chargeReceipts: [],
|
|
1107
|
+
pagination: {
|
|
1108
|
+
nextPageToken: null,
|
|
1109
|
+
paginationStatus: 'EXHAUSTED',
|
|
1110
|
+
paginationIncompleteReason: null,
|
|
1111
|
+
},
|
|
1112
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1113
|
+
billableCount: reconciledItems.length,
|
|
1114
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1115
|
+
durationMs,
|
|
1116
|
+
requestCount: Math.max(1, entityRes.rpcAttempts),
|
|
1117
|
+
}),
|
|
1118
|
+
});
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// Step 2: Fetch Creatives across selected verified advertiser entities
|
|
1122
|
+
const advertiserIds = entityRes.selectedEntities.map((e) => e.advertiserId);
|
|
1123
|
+
const creativesRes = await fetchAdvertiserCreatives({
|
|
1124
|
+
advertiserIds,
|
|
1125
|
+
entities: entityRes.selectedEntities,
|
|
1126
|
+
maxResults: input?.maxResults,
|
|
1127
|
+
maxPages: input?.maxPages,
|
|
1128
|
+
pageSize: input?.pageSize,
|
|
1129
|
+
pageToken: input?.pageToken,
|
|
1130
|
+
countryCode: input?.countryCode,
|
|
1131
|
+
formatFilter: input?.formatFilter,
|
|
1132
|
+
fetchImpl: hooks.fetchImpl,
|
|
1133
|
+
nowFn: nowMs,
|
|
1134
|
+
sleepFn,
|
|
1135
|
+
randomFn,
|
|
1136
|
+
timeoutMs: input?.timeoutMs,
|
|
1137
|
+
maxRetries: input?.maxRetries,
|
|
1138
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
let totalRpcRequests = entityRes.rpcAttempts + creativesRes.rpcAttempts;
|
|
1142
|
+
|
|
1143
|
+
if (!creativesRes.ok) {
|
|
1144
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1145
|
+
const isPreChargeDeadline =
|
|
1146
|
+
hasChargeHook &&
|
|
1147
|
+
nowMs() >= rpcDeadlineAtMs &&
|
|
1148
|
+
/deadline exceeded/i.test(String(creativesRes.reason || ''));
|
|
1149
|
+
return finalizeActorReturn({
|
|
1150
|
+
ok: false,
|
|
1151
|
+
status: isPreChargeDeadline ? 'DEADLINE_EXCEEDED_BEFORE_CHARGE' : creativesRes.status,
|
|
1152
|
+
reason: creativesRes.reason,
|
|
1153
|
+
billingMode,
|
|
1154
|
+
queryMeta: entityRes.queryMeta,
|
|
1155
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1156
|
+
items: reconciledItems,
|
|
1157
|
+
extractedCount: 0,
|
|
1158
|
+
deliveredCount,
|
|
1159
|
+
billableCount: reconciledItems.length,
|
|
1160
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1161
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1162
|
+
chargeReceipts: [],
|
|
1163
|
+
pagination: {
|
|
1164
|
+
nextPageToken: input?.pageToken || null,
|
|
1165
|
+
paginationStatus: isPreChargeDeadline
|
|
1166
|
+
? 'DEADLINE_EXCEEDED_HAS_MORE'
|
|
1167
|
+
: creativesRes.paginationStatus || creativesRes.status,
|
|
1168
|
+
paginationIncompleteReason: isPreChargeDeadline
|
|
1169
|
+
? 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
1170
|
+
: creativesRes.paginationIncompleteReason || creativesRes.reason || null,
|
|
1171
|
+
},
|
|
1172
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1173
|
+
billableCount: reconciledItems.length,
|
|
1174
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1175
|
+
durationMs,
|
|
1176
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1177
|
+
}),
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
const maxResultsCap = Math.min(200, Math.max(1, Number(input?.maxResults) || 40));
|
|
1182
|
+
|
|
1183
|
+
// Step 3: Strict Deduplication & Validation Gate before emitting Pay-per-Event charges
|
|
1184
|
+
const uniqueExtractedItems = [];
|
|
1185
|
+
const seenInRunIds = new Set();
|
|
1186
|
+
|
|
1187
|
+
for (const item of creativesRes.creatives) {
|
|
1188
|
+
if (
|
|
1189
|
+
!item ||
|
|
1190
|
+
!ADVERTISER_ID_RE.test(String(item.advertiserId || '')) ||
|
|
1191
|
+
!CREATIVE_ID_RE.test(String(item.creativeId || ''))
|
|
1192
|
+
) {
|
|
1193
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1194
|
+
return finalizeActorReturn({
|
|
1195
|
+
ok: false,
|
|
1196
|
+
status: 'SCHEMA_DRIFT_HALT',
|
|
1197
|
+
reason: 'Normalized creative record failed AR.../CR... identifier gate prior to billing',
|
|
1198
|
+
billingMode,
|
|
1199
|
+
queryMeta: entityRes.queryMeta,
|
|
1200
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1201
|
+
items: reconciledItems,
|
|
1202
|
+
extractedCount: 0,
|
|
1203
|
+
deliveredCount,
|
|
1204
|
+
billableCount: reconciledItems.length,
|
|
1205
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1206
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1207
|
+
chargeReceipts: [],
|
|
1208
|
+
pagination: {
|
|
1209
|
+
nextPageToken: null,
|
|
1210
|
+
paginationStatus: 'SCHEMA_DRIFT_HALT',
|
|
1211
|
+
paginationIncompleteReason: 'INVALID_CREATIVE_IDENTIFIER',
|
|
1212
|
+
},
|
|
1213
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1214
|
+
billableCount: reconciledItems.length,
|
|
1215
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1216
|
+
durationMs,
|
|
1217
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1218
|
+
}),
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
if (!seenInRunIds.has(item.creativeId)) {
|
|
1223
|
+
seenInRunIds.add(item.creativeId);
|
|
1224
|
+
uniqueExtractedItems.push(item);
|
|
1225
|
+
if (uniqueExtractedItems.length >= maxResultsCap) {
|
|
1226
|
+
break;
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
// Optional Step 3b: Enrich up to 10 creatives (default 5) via LookupService/GetCreativeById when includeCreativeDetails === true
|
|
1232
|
+
if (input?.includeCreativeDetails === true && uniqueExtractedItems.length > 0) {
|
|
1233
|
+
const requestedEnrichCap = Number(input?.maxEnrichItems ?? 5);
|
|
1234
|
+
const enrichCap = Math.min(
|
|
1235
|
+
10,
|
|
1236
|
+
Math.max(1, Number.isFinite(requestedEnrichCap) ? Math.floor(requestedEnrichCap) : 5),
|
|
1237
|
+
uniqueExtractedItems.length
|
|
1238
|
+
);
|
|
1239
|
+
let rateLimitedInEnrich = false;
|
|
1240
|
+
for (let i = 0; i < enrichCap; i += 1) {
|
|
1241
|
+
const targetItem = uniqueExtractedItems[i];
|
|
1242
|
+
if (rateLimitedInEnrich) {
|
|
1243
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1244
|
+
'NOT_ATTEMPTED_RATE_LIMITED'
|
|
1245
|
+
);
|
|
1246
|
+
continue;
|
|
1247
|
+
}
|
|
1248
|
+
if (nowMs() >= rpcDeadlineAtMs) {
|
|
1249
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1250
|
+
'NOT_ATTEMPTED_DEADLINE'
|
|
1251
|
+
);
|
|
1252
|
+
continue;
|
|
1253
|
+
}
|
|
1254
|
+
const detailRes = await getCreativeDetailsById({
|
|
1255
|
+
advertiserId: targetItem.advertiserId,
|
|
1256
|
+
creativeId: targetItem.creativeId,
|
|
1257
|
+
includeAdCopy: Boolean(input?.includeAdCopy),
|
|
1258
|
+
fetchImpl: hooks.fetchImpl,
|
|
1259
|
+
nowFn: nowMs,
|
|
1260
|
+
sleepFn,
|
|
1261
|
+
randomFn,
|
|
1262
|
+
timeoutMs: input?.timeoutMs,
|
|
1263
|
+
maxRetries: input?.maxRetries !== undefined ? input.maxRetries : 1,
|
|
1264
|
+
deadlineAtMs: rpcDeadlineAtMs,
|
|
1265
|
+
});
|
|
1266
|
+
totalRpcRequests += detailRes.rpcAttempts;
|
|
1267
|
+
if (detailRes.status === 'UPSTREAM_RATE_LIMITED') {
|
|
1268
|
+
rateLimitedInEnrich = true;
|
|
1269
|
+
targetItem.untrustedAdContent = createEmptyUntrustedAdContent(
|
|
1270
|
+
'NOT_ATTEMPTED_RATE_LIMITED'
|
|
1271
|
+
);
|
|
1272
|
+
continue;
|
|
1273
|
+
}
|
|
1274
|
+
if (detailRes.ok && detailRes.detail) {
|
|
1275
|
+
targetItem.creativeDetail = detailRes.detail;
|
|
1276
|
+
if (detailRes.detail.untrustedAdContent) {
|
|
1277
|
+
targetItem.untrustedAdContent = detailRes.detail.untrustedAdContent;
|
|
1278
|
+
if (
|
|
1279
|
+
detailRes.detail.untrustedAdContent.extractionStatus === 'UPSTREAM_RATE_LIMITED' ||
|
|
1280
|
+
detailRes.detail.untrustedAdContent.contentJsFetchStatus === 'UPSTREAM_RATE_LIMITED'
|
|
1281
|
+
) {
|
|
1282
|
+
rateLimitedInEnrich = true;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
const extractedCount = uniqueExtractedItems.length;
|
|
1290
|
+
|
|
1291
|
+
// Filter out items already charged, delivered, or marked unknown in prior checkpoint
|
|
1292
|
+
const newBillableCandidates = uniqueExtractedItems.filter(
|
|
1293
|
+
(item) =>
|
|
1294
|
+
!chargedSet.has(item.creativeId) &&
|
|
1295
|
+
!deliveredSet.has(item.creativeId) &&
|
|
1296
|
+
!unknownOutcomeSet.has(item.creativeId)
|
|
1297
|
+
);
|
|
1298
|
+
|
|
1299
|
+
if (newBillableCandidates.length === 0 && reconciledItems.length === 0) {
|
|
1300
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1301
|
+
return finalizeActorReturn({
|
|
1302
|
+
ok: true,
|
|
1303
|
+
status: 'EMPTY_RESULT',
|
|
1304
|
+
billingMode,
|
|
1305
|
+
queryMeta: entityRes.queryMeta,
|
|
1306
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1307
|
+
items: [],
|
|
1308
|
+
extractedCount,
|
|
1309
|
+
deliveredCount,
|
|
1310
|
+
billableCount: 0,
|
|
1311
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1312
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1313
|
+
chargeReceipts: [],
|
|
1314
|
+
pagination: {
|
|
1315
|
+
nextPageToken: creativesRes.nextPageToken,
|
|
1316
|
+
paginationStatus: creativesRes.paginationStatus,
|
|
1317
|
+
paginationIncompleteReason: creativesRes.paginationIncompleteReason,
|
|
1318
|
+
pagesFetched: creativesRes.pagesFetched,
|
|
1319
|
+
},
|
|
1320
|
+
cursorResumeContext: creativesRes.cursorResumeContext,
|
|
1321
|
+
telemetry: computeRunEconomicsTelemetry({
|
|
1322
|
+
billableCount: 0,
|
|
1323
|
+
chargedEventCount: hasChargeHook ? chargedSet.size : 0,
|
|
1324
|
+
durationMs,
|
|
1325
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1326
|
+
}),
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
// Step 4: Chunked Dataset Push (25 items/chunk) & Crash-Safe Event Charging
|
|
1331
|
+
const chunkSize = ACTOR_PRICING_CONTRACT.chunkSize;
|
|
1332
|
+
const chargeReceipts = [];
|
|
1333
|
+
const newlyDeliveredItems = [];
|
|
1334
|
+
let chargedEventCount = hasChargeHook ? chargedSet.size : 0;
|
|
1335
|
+
let finalStatus = 'OK';
|
|
1336
|
+
let finalOk = true;
|
|
1337
|
+
let haltReason = undefined;
|
|
1338
|
+
const runtimeWarnings = Array.isArray(creativesRes.warnings) ? [...creativesRes.warnings] : [];
|
|
1339
|
+
const persistCheckpointPostCharge = async () => {
|
|
1340
|
+
try {
|
|
1341
|
+
await persistCheckpoint();
|
|
1342
|
+
} catch (cpErr) {
|
|
1343
|
+
runtimeWarnings.push(
|
|
1344
|
+
`Post-charge checkpoint write warning: ${cpErr?.message || String(cpErr)}`
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
};
|
|
1348
|
+
|
|
1349
|
+
for (let offset = 0; offset < newBillableCandidates.length; offset += chunkSize) {
|
|
1350
|
+
const effectiveChunkDeadlineMs = hasChargeHook
|
|
1351
|
+
? deadlineAtMs - step4ReserveMs
|
|
1352
|
+
: deadlineAtMs;
|
|
1353
|
+
if (nowMs() >= effectiveChunkDeadlineMs) {
|
|
1354
|
+
finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
|
|
1355
|
+
finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
|
|
1356
|
+
haltReason = 'Run deadline reserve reached prior to charging next chunk; halted cleanly before billing';
|
|
1357
|
+
break;
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
const unchargedChunk = newBillableCandidates
|
|
1361
|
+
.slice(offset, offset + chunkSize)
|
|
1362
|
+
.filter(
|
|
1363
|
+
(c) =>
|
|
1364
|
+
!chargedSet.has(c.creativeId) &&
|
|
1365
|
+
!deliveredSet.has(c.creativeId) &&
|
|
1366
|
+
!unknownOutcomeSet.has(c.creativeId)
|
|
1367
|
+
);
|
|
1368
|
+
|
|
1369
|
+
if (unchargedChunk.length === 0) {
|
|
1370
|
+
continue;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
const chunkIds = unchargedChunk.map((c) => c.creativeId);
|
|
1374
|
+
const idempotencyKey = computeChunkIdempotencyKey(runId, chunkIds);
|
|
1375
|
+
|
|
1376
|
+
if (!hasChargeHook) {
|
|
1377
|
+
// Local / MCP unbilled execution mode: deliver items with chargedEventCount = 0
|
|
1378
|
+
const pushRes = await pushDataWithRetry(unchargedChunk);
|
|
1379
|
+
if (!pushRes.ok) {
|
|
1380
|
+
await persistCheckpointPostCharge();
|
|
1381
|
+
finalOk = false;
|
|
1382
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1383
|
+
haltReason = `Dataset pushData failed in local mode: ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1384
|
+
break;
|
|
1385
|
+
}
|
|
1386
|
+
for (const c of unchargedChunk) {
|
|
1387
|
+
deliveredSet.add(c.creativeId);
|
|
1388
|
+
newlyDeliveredItems.push(c);
|
|
1389
|
+
}
|
|
1390
|
+
deliveredCount = deliveredSet.size;
|
|
1391
|
+
await persistCheckpointPostCharge();
|
|
1392
|
+
continue;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
// Step 4.1: Persist inFlightIds and stagedItemsById BEFORE invoking chargeEvent
|
|
1396
|
+
inFlightSet = new Set(chunkIds);
|
|
1397
|
+
for (const c of unchargedChunk) {
|
|
1398
|
+
stagedItemsById[c.creativeId] = c;
|
|
1399
|
+
}
|
|
1400
|
+
try {
|
|
1401
|
+
await persistCheckpoint();
|
|
1402
|
+
} catch (preCpErr) {
|
|
1403
|
+
inFlightSet = new Set();
|
|
1404
|
+
finalOk = false;
|
|
1405
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1406
|
+
haltReason = `Pre-charge checkpoint write failed before billing: ${preCpErr?.message || String(preCpErr)}`;
|
|
1407
|
+
break;
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
if (nowMs() >= effectiveChunkDeadlineMs) {
|
|
1411
|
+
inFlightSet = new Set();
|
|
1412
|
+
await persistCheckpointPostCharge();
|
|
1413
|
+
finalOk = newlyDeliveredItems.length > 0 || reconciledItems.length > 0;
|
|
1414
|
+
finalStatus = 'DEADLINE_EXCEEDED_BEFORE_CHARGE';
|
|
1415
|
+
haltReason =
|
|
1416
|
+
'Run deadline reserve reached immediately after pre-charge checkpoint write; cleared inFlightIds and halted before billing';
|
|
1417
|
+
break;
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
const receipt = {
|
|
1421
|
+
idempotencyKey,
|
|
1422
|
+
eventName: ACTOR_PRICING_CONTRACT.eventName,
|
|
1423
|
+
count: unchargedChunk.length,
|
|
1424
|
+
unitPriceUsdc: ACTOR_PRICING_CONTRACT.priceUsdc,
|
|
1425
|
+
chunkAmountUsdc: Number(
|
|
1426
|
+
(unchargedChunk.length * ACTOR_PRICING_CONTRACT.priceUsdc).toFixed(6)
|
|
1427
|
+
),
|
|
1428
|
+
creativeIds: chunkIds,
|
|
1429
|
+
};
|
|
1430
|
+
|
|
1431
|
+
let chargeResult;
|
|
1432
|
+
try {
|
|
1433
|
+
chargeResult = await hooks.chargeEvent(receipt);
|
|
1434
|
+
} catch (chargeErr) {
|
|
1435
|
+
for (const id of chunkIds) {
|
|
1436
|
+
unknownOutcomeSet.add(id);
|
|
1437
|
+
}
|
|
1438
|
+
inFlightSet = new Set();
|
|
1439
|
+
await persistCheckpointPostCharge();
|
|
1440
|
+
finalOk = false;
|
|
1441
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1442
|
+
haltReason = `chargeEvent threw error: ${chargeErr?.message || String(chargeErr)}`;
|
|
1443
|
+
break;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// Step 4.2: Strict ChargeResult validation (never fallback to ?? unchargedChunk.length)
|
|
1447
|
+
const chargedCount = chargeResult?.chargedCount;
|
|
1448
|
+
const eventChargeLimitReached = Boolean(chargeResult?.eventChargeLimitReached);
|
|
1449
|
+
|
|
1450
|
+
if (
|
|
1451
|
+
!chargeResult ||
|
|
1452
|
+
typeof chargeResult !== 'object' ||
|
|
1453
|
+
!Number.isInteger(chargedCount) ||
|
|
1454
|
+
chargedCount < 0 ||
|
|
1455
|
+
chargedCount > unchargedChunk.length
|
|
1456
|
+
) {
|
|
1457
|
+
for (const id of chunkIds) {
|
|
1458
|
+
unknownOutcomeSet.add(id);
|
|
1459
|
+
}
|
|
1460
|
+
inFlightSet = new Set();
|
|
1461
|
+
await persistCheckpointPostCharge();
|
|
1462
|
+
finalOk = false;
|
|
1463
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1464
|
+
haltReason =
|
|
1465
|
+
'chargeEvent returned invalid or missing integer chargedCount (transitioned chunk to UNKNOWN_CHARGE_OUTCOME)';
|
|
1466
|
+
break;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
// Step 4.3: Branch on validated chargedCount
|
|
1470
|
+
if (chargedCount === unchargedChunk.length) {
|
|
1471
|
+
// Full chunk charged: persist chargedIds immediately BEFORE pushData (best-effort so KVS glitch never blocks delivery)
|
|
1472
|
+
for (const id of chunkIds) {
|
|
1473
|
+
chargedSet.add(id);
|
|
1474
|
+
}
|
|
1475
|
+
inFlightSet = new Set();
|
|
1476
|
+
chargedEventCount = chargedSet.size;
|
|
1477
|
+
await persistCheckpointPostCharge();
|
|
1478
|
+
|
|
1479
|
+
const pushRes = await pushDataWithRetry(unchargedChunk);
|
|
1480
|
+
if (!pushRes.ok) {
|
|
1481
|
+
chargeReceipts.push({
|
|
1482
|
+
...receipt,
|
|
1483
|
+
chargedCount,
|
|
1484
|
+
eventChargeLimitReached,
|
|
1485
|
+
});
|
|
1486
|
+
finalOk = false;
|
|
1487
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1488
|
+
haltReason = `Dataset pushData failed after charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1489
|
+
break;
|
|
1490
|
+
}
|
|
1491
|
+
for (const c of unchargedChunk) {
|
|
1492
|
+
deliveredSet.add(c.creativeId);
|
|
1493
|
+
newlyDeliveredItems.push(c);
|
|
1494
|
+
}
|
|
1495
|
+
deliveredCount = deliveredSet.size;
|
|
1496
|
+
await persistCheckpointPostCharge();
|
|
1497
|
+
|
|
1498
|
+
chargeReceipts.push({
|
|
1499
|
+
...receipt,
|
|
1500
|
+
chargedCount,
|
|
1501
|
+
eventChargeLimitReached,
|
|
1502
|
+
});
|
|
1503
|
+
|
|
1504
|
+
if (eventChargeLimitReached) {
|
|
1505
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
1506
|
+
break;
|
|
1507
|
+
}
|
|
1508
|
+
} else if (chargedCount === 0) {
|
|
1509
|
+
inFlightSet = new Set();
|
|
1510
|
+
await persistCheckpointPostCharge();
|
|
1511
|
+
chargeReceipts.push({
|
|
1512
|
+
...receipt,
|
|
1513
|
+
chargedCount: 0,
|
|
1514
|
+
eventChargeLimitReached,
|
|
1515
|
+
});
|
|
1516
|
+
if (eventChargeLimitReached) {
|
|
1517
|
+
finalOk = true;
|
|
1518
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
1519
|
+
} else {
|
|
1520
|
+
finalOk = false;
|
|
1521
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1522
|
+
haltReason = 'chargeEvent returned chargedCount = 0 without eventChargeLimitReached';
|
|
1523
|
+
}
|
|
1524
|
+
break;
|
|
1525
|
+
} else {
|
|
1526
|
+
// Partial chunk charge (0 < chargedCount < unchargedChunk.length)
|
|
1527
|
+
const explicitChargedIds = Array.isArray(chargeResult.chargedCreativeIds)
|
|
1528
|
+
? chargeResult.chargedCreativeIds.filter((id) => chunkIds.includes(id))
|
|
1529
|
+
: null;
|
|
1530
|
+
|
|
1531
|
+
if (explicitChargedIds && explicitChargedIds.length === chargedCount) {
|
|
1532
|
+
for (const id of explicitChargedIds) {
|
|
1533
|
+
chargedSet.add(id);
|
|
1534
|
+
}
|
|
1535
|
+
inFlightSet = new Set();
|
|
1536
|
+
chargedEventCount = chargedSet.size;
|
|
1537
|
+
await persistCheckpointPostCharge();
|
|
1538
|
+
|
|
1539
|
+
const paidSubset = unchargedChunk.filter((c) => explicitChargedIds.includes(c.creativeId));
|
|
1540
|
+
const pushRes = await pushDataWithRetry(paidSubset);
|
|
1541
|
+
if (!pushRes.ok) {
|
|
1542
|
+
chargeReceipts.push({
|
|
1543
|
+
...receipt,
|
|
1544
|
+
chargedCount,
|
|
1545
|
+
chargedCreativeIds: explicitChargedIds,
|
|
1546
|
+
eventChargeLimitReached,
|
|
1547
|
+
});
|
|
1548
|
+
finalOk = false;
|
|
1549
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1550
|
+
haltReason = `Dataset pushData failed after partial charge (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1551
|
+
break;
|
|
1552
|
+
}
|
|
1553
|
+
for (const c of paidSubset) {
|
|
1554
|
+
deliveredSet.add(c.creativeId);
|
|
1555
|
+
newlyDeliveredItems.push(c);
|
|
1556
|
+
}
|
|
1557
|
+
deliveredCount = deliveredSet.size;
|
|
1558
|
+
await persistCheckpointPostCharge();
|
|
1559
|
+
|
|
1560
|
+
chargeReceipts.push({
|
|
1561
|
+
...receipt,
|
|
1562
|
+
chargedCount,
|
|
1563
|
+
chargedCreativeIds: explicitChargedIds,
|
|
1564
|
+
eventChargeLimitReached,
|
|
1565
|
+
});
|
|
1566
|
+
|
|
1567
|
+
finalOk = eventChargeLimitReached;
|
|
1568
|
+
finalStatus = eventChargeLimitReached ? 'CHARGE_LIMIT_REACHED' : 'PARTIAL_BILLING_HALT';
|
|
1569
|
+
if (!eventChargeLimitReached) {
|
|
1570
|
+
haltReason = `Partial chunk charge (${chargedCount}/${unchargedChunk.length}) without charge limit flag`;
|
|
1571
|
+
}
|
|
1572
|
+
break;
|
|
1573
|
+
} else if (eventChargeLimitReached) {
|
|
1574
|
+
// Standard Apify SDK ChargeResult when hitting user's max charge ceiling:
|
|
1575
|
+
// Attribute charge deterministically to the ordered prefix `unchargedChunk.slice(0, chargedCount)`
|
|
1576
|
+
// so the customer receives every item they paid for, and leave the remaining tail uncharged.
|
|
1577
|
+
const paidPrefix = unchargedChunk.slice(0, chargedCount);
|
|
1578
|
+
const paidPrefixIds = paidPrefix.map((c) => c.creativeId);
|
|
1579
|
+
for (const id of paidPrefixIds) {
|
|
1580
|
+
chargedSet.add(id);
|
|
1581
|
+
}
|
|
1582
|
+
inFlightSet = new Set();
|
|
1583
|
+
chargedEventCount = chargedSet.size;
|
|
1584
|
+
await persistCheckpointPostCharge();
|
|
1585
|
+
|
|
1586
|
+
const pushRes = await pushDataWithRetry(paidPrefix);
|
|
1587
|
+
if (!pushRes.ok) {
|
|
1588
|
+
chargeReceipts.push({
|
|
1589
|
+
...receipt,
|
|
1590
|
+
chargedCount,
|
|
1591
|
+
chargedCreativeIds: paidPrefixIds,
|
|
1592
|
+
eventChargeLimitReached: true,
|
|
1593
|
+
});
|
|
1594
|
+
finalOk = false;
|
|
1595
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1596
|
+
haltReason = `Dataset pushData failed after charge limit prefix (paid items preserved in KVS checkpoint stagedItemsById): ${pushRes.error?.message || String(pushRes.error)}`;
|
|
1597
|
+
break;
|
|
1598
|
+
}
|
|
1599
|
+
for (const c of paidPrefix) {
|
|
1600
|
+
deliveredSet.add(c.creativeId);
|
|
1601
|
+
newlyDeliveredItems.push(c);
|
|
1602
|
+
}
|
|
1603
|
+
deliveredCount = deliveredSet.size;
|
|
1604
|
+
await persistCheckpointPostCharge();
|
|
1605
|
+
|
|
1606
|
+
chargeReceipts.push({
|
|
1607
|
+
...receipt,
|
|
1608
|
+
chargedCount,
|
|
1609
|
+
chargedCreativeIds: paidPrefixIds,
|
|
1610
|
+
eventChargeLimitReached: true,
|
|
1611
|
+
});
|
|
1612
|
+
|
|
1613
|
+
finalOk = true;
|
|
1614
|
+
finalStatus = 'CHARGE_LIMIT_REACHED';
|
|
1615
|
+
break;
|
|
1616
|
+
} else {
|
|
1617
|
+
// Unexpected partial count without eventChargeLimitReached and without explicit IDs ->
|
|
1618
|
+
// move chunk to unknownOutcomeIds and halt fail-closed
|
|
1619
|
+
for (const id of chunkIds) {
|
|
1620
|
+
unknownOutcomeSet.add(id);
|
|
1621
|
+
}
|
|
1622
|
+
inFlightSet = new Set();
|
|
1623
|
+
chargedEventCount = chargedSet.size + chargedCount;
|
|
1624
|
+
await persistCheckpointPostCharge();
|
|
1625
|
+
|
|
1626
|
+
chargeReceipts.push({
|
|
1627
|
+
...receipt,
|
|
1628
|
+
chargedCount,
|
|
1629
|
+
eventChargeLimitReached: false,
|
|
1630
|
+
});
|
|
1631
|
+
|
|
1632
|
+
finalOk = false;
|
|
1633
|
+
finalStatus = 'PARTIAL_BILLING_HALT';
|
|
1634
|
+
haltReason = `Ambiguous partial chunk charge (${chargedCount}/${unchargedChunk.length}) without chargedCreativeIds attribution or eventChargeLimitReached`;
|
|
1635
|
+
break;
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
// Step 5: Rewind pagination cursor if billing halted or reached charge limit before delivering all extracted items
|
|
1641
|
+
let effectiveNextPageToken = creativesRes.nextPageToken;
|
|
1642
|
+
let effectivePaginationStatus = creativesRes.paginationStatus;
|
|
1643
|
+
let effectiveIncompleteReason = creativesRes.paginationIncompleteReason;
|
|
1644
|
+
const undeliveredCount = Math.max(0, newBillableCandidates.length - newlyDeliveredItems.length);
|
|
1645
|
+
|
|
1646
|
+
if (undeliveredCount > 0) {
|
|
1647
|
+
const firstUndeliveredItem = newBillableCandidates.find((c) => !deliveredSet.has(c.creativeId));
|
|
1648
|
+
const firstUndeliveredExtractedIdx = Array.isArray(creativesRes.creatives)
|
|
1649
|
+
? creativesRes.creatives.findIndex((c) => c.creativeId === firstUndeliveredItem?.creativeId)
|
|
1650
|
+
: -1;
|
|
1651
|
+
const effectiveDeliveredPrefixLen =
|
|
1652
|
+
firstUndeliveredExtractedIdx >= 0 ? firstUndeliveredExtractedIdx : newlyDeliveredItems.length;
|
|
1653
|
+
effectiveNextPageToken = computeRewoundCursorForDeliveredPrefix(
|
|
1654
|
+
creativesRes.cursorResumeContext,
|
|
1655
|
+
effectiveDeliveredPrefixLen,
|
|
1656
|
+
creativesRes.nextPageToken
|
|
1657
|
+
);
|
|
1658
|
+
effectivePaginationStatus =
|
|
1659
|
+
finalStatus === 'CHARGE_LIMIT_REACHED'
|
|
1660
|
+
? 'CHARGE_LIMIT_REACHED_HAS_MORE'
|
|
1661
|
+
: finalStatus === 'DEADLINE_EXCEEDED_BEFORE_CHARGE'
|
|
1662
|
+
? 'DEADLINE_EXCEEDED_HAS_MORE'
|
|
1663
|
+
: 'PARTIAL_BILLING_HALT';
|
|
1664
|
+
effectiveIncompleteReason = finalStatus;
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
const paginationMeta = {
|
|
1668
|
+
nextPageToken: effectiveNextPageToken,
|
|
1669
|
+
paginationStatus: effectivePaginationStatus,
|
|
1670
|
+
paginationIncompleteReason: effectiveIncompleteReason,
|
|
1671
|
+
pagesFetched: creativesRes.pagesFetched,
|
|
1672
|
+
...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
|
|
1673
|
+
...(undeliveredCount > 0 ? { undeliveredCount } : {}),
|
|
1674
|
+
};
|
|
1675
|
+
|
|
1676
|
+
// Merge reconciled items + newly delivered items with strict creativeId deduplication
|
|
1677
|
+
const allOutputItems = [];
|
|
1678
|
+
const outputSeenIds = new Set();
|
|
1679
|
+
for (const item of [...reconciledItems, ...newlyDeliveredItems]) {
|
|
1680
|
+
if (item?.creativeId && !outputSeenIds.has(item.creativeId)) {
|
|
1681
|
+
outputSeenIds.add(item.creativeId);
|
|
1682
|
+
allOutputItems.push(item);
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
const simulatedBillableCount = hasChargeHook ? chargedEventCount : allOutputItems.length;
|
|
1687
|
+
const durationMs = Math.max(1, nowMs() - startedAtMs);
|
|
1688
|
+
const telemetry = computeRunEconomicsTelemetry({
|
|
1689
|
+
billableCount: simulatedBillableCount,
|
|
1690
|
+
chargedEventCount,
|
|
1691
|
+
durationMs,
|
|
1692
|
+
requestCount: Math.max(1, totalRpcRequests),
|
|
1693
|
+
});
|
|
1694
|
+
|
|
1695
|
+
return finalizeActorReturn({
|
|
1696
|
+
ok: finalOk,
|
|
1697
|
+
status: finalStatus,
|
|
1698
|
+
reason: haltReason,
|
|
1699
|
+
billingMode,
|
|
1700
|
+
queryMeta: entityRes.queryMeta,
|
|
1701
|
+
selectedEntities: entityRes.selectedEntities,
|
|
1702
|
+
items: allOutputItems,
|
|
1703
|
+
extractedCount,
|
|
1704
|
+
deliveredCount,
|
|
1705
|
+
billableCount: simulatedBillableCount,
|
|
1706
|
+
chargedEventCount,
|
|
1707
|
+
unknownOutcomeIds: Array.from(unknownOutcomeSet),
|
|
1708
|
+
chargeReceipts,
|
|
1709
|
+
pagination: paginationMeta,
|
|
1710
|
+
...(runtimeWarnings.length > 0 ? { warnings: runtimeWarnings } : {}),
|
|
1711
|
+
cursorResumeContext: creativesRes.cursorResumeContext,
|
|
1712
|
+
telemetry,
|
|
1713
|
+
});
|
|
1714
|
+
}
|