@aglyn/tenant-data-admin 1.0.0-beta.284 → 1.0.0-beta.287

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,8 +17,9 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
17
17
  */ import { AD_AUDIENCE_OPT_OUT_FIELD, readAdAudienceOptOut } from "@aglyn/aglyn/app-utils/ad-audience";
18
18
  import { ACCOUNT_ACQUISITION_FIELD } from "@aglyn/aglyn/app-utils/account-acquisition";
19
19
  import { isAccountBanLockdownReason, LOCKDOWNS_COLLECTION } from "@aglyn/aglyn/app-utils/lockdown";
20
- import { FieldPath } from "firebase-admin/firestore";
21
- import { AD_AUDIENCE_EXCLUSIONS, AD_AUDIENCE_PROVIDERS, adAudienceExclusion, buildGoogleMemberRequests, buildGoogleTokenRequest, buildLinkedInUserRequests, buildMetaReplaceRequests, diffAudience, hashAudienceEmail, isSuspiciousShrink, linkedInRefusedCount, resolveAccountCountry } from "./ad-audience-sync.js";
20
+ import { resolveEffectivePlan } from "@aglyn/aglyn/app-utils/plan-entitlements";
21
+ import { FieldPath, Timestamp } from "firebase-admin/firestore";
22
+ import { AD_AUDIENCE_EXCLUSIONS, AD_AUDIENCE_LAPSE_MS, AD_AUDIENCE_PROVIDERS, AD_AUDIENCE_STAGES, adAudienceExclusion, adAudienceStages, buildGoogleMemberRequests, buildGoogleTokenRequest, buildLinkedInUserRequests, buildMetaReplaceRequests, diffAudience, hashAudienceEmail, instantMs, isOwnPublishedScreen, isRealActivityAction, isSuspiciousShrink, linkedInRefusedCount, resolveAccountCountry } from "./ad-audience-sync.js";
22
23
  /**
23
24
  * THE ACCOUNT-HOLDER AD AUDIENCE (AGL-3780) — reading, sending, recording.
24
25
  *
@@ -35,11 +36,24 @@ import { AD_AUDIENCE_EXCLUSIONS, AD_AUDIENCE_PROVIDERS, adAudienceExclusion, bui
35
36
  * the only way to take it out of a Google list or a LinkedIn segment is the
36
37
  * hash this sync sent it under. Admin-SDK only; no client rule opens either.
37
38
  * No address is stored, and neither addresses nor hashes are ever logged.
39
+ *
40
+ * A STAGE list (AGL-4072) keeps the same two things under its own document,
41
+ * `adAudienceSync/{provider}--{stage}`, so each stage has its own ledger, its
42
+ * own diff and its own shrink guard, and nothing one list does can touch what
43
+ * another was sent.
38
44
  */ export const AD_AUDIENCE_SYNC_COLLECTION = 'adAudienceSync';
39
45
  export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
40
46
  /** Hashes per member chunk: 5,000 × 64 bytes stays far under a document's 1 MiB. */ const MEMBER_CHUNK = 5000;
41
47
  /** Auth pages walked per run (1,000 accounts each). */ const MAX_AUTH_PAGES = 500;
42
- /** Every account, with the facts eligibility reads. */ export async function readAdAudienceAccounts(deps) {
48
+ /** Page documents read per page of the published-page scan. */ const SCREEN_PAGE = 5000;
49
+ /** Change-log entries read per page while looking for one the person made. */ const ACTIVITY_PAGE = 25;
50
+ /** Pages of change log read per account before it is called quiet. */ const ACTIVITY_PAGES = 4;
51
+ /** Accounts whose recent activity is read at once. */ const ACTIVITY_CONCURRENCY = 20;
52
+ /** The ledger document of one list: the provider's own, or `{provider}--{stage}`. */ export function adAudienceLedgerId(provider, stage) {
53
+ return stage ? `${provider}--${stage}` : provider;
54
+ }
55
+ /** Every account, with the facts eligibility and the stage lists read. */ export async function readAdAudienceAccounts(deps) {
56
+ var _deps_recentActivity, _deps_internalDomains;
43
57
  const db = deps.firestore;
44
58
  // Profiles: the opt-out and the sign-up country, in one query.
45
59
  const profiles = await db.collection('users').select(AD_AUDIENCE_OPT_OUT_FIELD, `${ACCOUNT_ACQUISITION_FIELD}.geo.country`).get();
@@ -53,19 +67,42 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
53
67
  const geo = (_data_ACCOUNT_ACQUISITION_FIELD = data[ACCOUNT_ACQUISITION_FIELD]) == null ? void 0 : _data_ACCOUNT_ACQUISITION_FIELD.geo;
54
68
  if (typeof (geo == null ? void 0 : geo.country) === 'string') signupCountry.set(doc.id, geo.country);
55
69
  }
56
- // Billing countries of the workspaces each account owns.
57
- const orgs = await db.collection('orgs').select('ownerUid', 'contact.address.country').get();
70
+ // The workspaces each account owns, in one query: their billing countries,
71
+ // whether any is on a paid plan, and the sites they hold. The plan is
72
+ // `resolveEffectivePlan`'s, the one every entitlement reads — a canceled
73
+ // subscription is Free, a staff comp is the plan it grants — and it needs
74
+ // only the org document's own fields (the derived `billingStatus` included).
75
+ const orgs = await db.collection('orgs').select('ownerUid', 'contact.address.country', 'plan', 'billingStatus', 'subscription.status', 'entitlements.planComp', 'hosts').get();
58
76
  const billing = new Map();
77
+ const paidOwners = new Set();
78
+ const hostOwner = new Map();
59
79
  for (const doc of orgs.docs){
60
- var _billing_get;
80
+ var _doc_data1;
61
81
  const owner = doc.get('ownerUid');
82
+ if (typeof owner !== 'string' || !owner) continue;
62
83
  const country = doc.get('contact.address.country');
63
- if (typeof owner !== 'string' || !owner || typeof country !== 'string' || !country) continue;
64
- billing.set(owner, [
65
- ...(_billing_get = billing.get(owner)) != null ? _billing_get : [],
66
- country
67
- ]);
84
+ if (typeof country === 'string' && country) {
85
+ var _billing_get;
86
+ billing.set(owner, [
87
+ ...(_billing_get = billing.get(owner)) != null ? _billing_get : [],
88
+ country
89
+ ]);
90
+ }
91
+ const data = (_doc_data1 = doc.data()) != null ? _doc_data1 : {};
92
+ if (resolveEffectivePlan(data) !== 'free') paidOwners.add(owner);
93
+ const hosts = data['hosts'];
94
+ if (hosts && typeof hosts === 'object') {
95
+ for (const hostId of Object.keys(hosts))hostOwner.set(hostId, owner);
96
+ }
68
97
  }
98
+ // The stage facts never hold up the all-holders list: a read that fails
99
+ // leaves the accounts in no stage tonight, which every stage list's shrink
100
+ // hold (or its refusal to send an empty replace) keeps from reaching a
101
+ // provider, and the main list is synced as ever.
102
+ const publishedOwners = await readOwnPublishedOwners(db, hostOwner, deps.now).catch((error)=>{
103
+ console.error('[ad-audience-sync] the published-page read failed; no stage tonight', error);
104
+ return null;
105
+ });
69
106
  // Account locks: `lockdowns/user--{uid}`, by document id range.
70
107
  const locks = await db.collection(LOCKDOWNS_COLLECTION).where(FieldPath.documentId(), '>=', 'user--').where(FieldPath.documentId(), '<', 'user-.').get();
71
108
  const locked = new Set();
@@ -83,8 +120,8 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
83
120
  const listed = await deps.listUserPage(1000, pageToken);
84
121
  for (const { record } of listed.users){
85
122
  var _record_email, _billing_get1, _signupCountry_get;
86
- var _record_customClaims;
87
- accounts.push({
123
+ var _record_customClaims, _record_metadata;
124
+ accounts.push(_extends({
88
125
  uid: record.uid,
89
126
  email: (_record_email = record.email) != null ? _record_email : null,
90
127
  emailVerified: Boolean(record.emailVerified),
@@ -95,7 +132,14 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
95
132
  banned: false,
96
133
  billingCountries: (_billing_get1 = billing.get(record.uid)) != null ? _billing_get1 : [],
97
134
  signupCountry: (_signupCountry_get = signupCountry.get(record.uid)) != null ? _signupCountry_get : null
98
- });
135
+ }, publishedOwners ? {
136
+ stageFacts: {
137
+ createdAtMs: instantMs((_record_metadata = record.metadata) == null ? void 0 : _record_metadata.creationTime),
138
+ ownsPaidWorkspace: paidOwners.has(record.uid),
139
+ ownsPublishedSite: publishedOwners.has(record.uid),
140
+ lastActivityAtMs: null
141
+ }
142
+ } : {}));
99
143
  }
100
144
  pageToken = (_listed_nextPageToken = listed.nextPageToken) != null ? _listed_nextPageToken : undefined;
101
145
  if (!pageToken) break;
@@ -108,11 +152,101 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
108
152
  for (const account of accounts){
109
153
  if (account.email && banned.has(account.email.trim().toLowerCase())) account.banned = true;
110
154
  }
155
+ // Recent activity, for the only accounts it can change anything for: those
156
+ // in the audience and old enough to lapse. Everyone else's change log is
157
+ // never read, which keeps the nightly cost to the accounts that matter.
158
+ const since = deps.now - AD_AUDIENCE_LAPSE_MS;
159
+ const recentActivity = (_deps_recentActivity = deps.recentActivity) != null ? _deps_recentActivity : (uid, sinceMs)=>readRecentActivityAtMs(db, uid, sinceMs);
160
+ const internalDomains = (_deps_internalDomains = deps.internalDomains) != null ? _deps_internalDomains : [];
161
+ const candidates = accounts.filter((account)=>{
162
+ var _account_stageFacts;
163
+ const created = (_account_stageFacts = account.stageFacts) == null ? void 0 : _account_stageFacts.createdAtMs;
164
+ return typeof created === 'number' && created < since && !adAudienceExclusion(account, {
165
+ internalDomains
166
+ });
167
+ });
168
+ let activityFailures = 0;
169
+ for(let index = 0; index < candidates.length; index += ACTIVITY_CONCURRENCY){
170
+ await Promise.all(candidates.slice(index, index + ACTIVITY_CONCURRENCY).map(async (account)=>{
171
+ try {
172
+ const at = await recentActivity(account.uid, since);
173
+ if (account.stageFacts) account.stageFacts.lastActivityAtMs = at;
174
+ } catch (unused) {
175
+ // Unknown is not quiet: out of every stage tonight, never lapsed by mistake.
176
+ delete account.stageFacts;
177
+ activityFailures += 1;
178
+ }
179
+ }));
180
+ }
181
+ if (activityFailures) {
182
+ console.error(`[ad-audience-sync] recent activity unread for ${activityFailures} accounts; in no stage tonight`);
183
+ }
111
184
  return {
112
185
  accounts
113
186
  };
114
187
  }
115
- /** The eligible accounts and the counts of everyone left out. */ export function selectAdAudience(accounts, internalDomains) {
188
+ /**
189
+ * The owners of a site with a page of their own published, from one scan of
190
+ * every page document (`hosts/{hostId}/screens/{id}`), paged by path and
191
+ * reading only the fields the test needs. One query instead of one per site;
192
+ * a page whose site no workspace holds counts for nobody.
193
+ */ async function readOwnPublishedOwners(db, hostOwner, now) {
194
+ const owners = new Set();
195
+ if (!hostOwner.size) return owners;
196
+ const base = db.collectionGroup('screens').orderBy(FieldPath.documentId()).select('publishedAt', 'deletedAt', 'createdAt', 'updatedAt', 'createdBy').limit(SCREEN_PAGE);
197
+ let after;
198
+ for(;;){
199
+ const page = await (after ? base.startAfter(after) : base).get();
200
+ for (const doc of page.docs){
201
+ var _doc_data;
202
+ // `hosts/{hostId}/screens/{screenId}`; any other `screens` is not a site's.
203
+ const [root, hostId] = doc.ref.path.split('/');
204
+ const owner = root === 'hosts' ? hostOwner.get(hostId) : undefined;
205
+ if (!owner || owners.has(owner)) continue;
206
+ if (isOwnPublishedScreen((_doc_data = doc.data()) != null ? _doc_data : {}, now)) owners.add(owner);
207
+ }
208
+ if (page.docs.length < SCREEN_PAGE) break;
209
+ after = page.docs[page.docs.length - 1];
210
+ }
211
+ return owners;
212
+ }
213
+ /**
214
+ * The newest thing one account DID since `sinceMs`, or null when nothing was
215
+ * recorded. Two signals, both read through collection-group indexes that
216
+ * already exist for the staff activity feed and the AI job list:
217
+ *
218
+ * - **Change-log entries** under the account's id, on any site's or
219
+ * workspace's log (`hosts/{id}/activity`, `orgs/{id}/activity`): page
220
+ * edits, publishes, form and CRM actions, settings. Entries the platform
221
+ * writes under the person's id without them acting
222
+ * (`isRealActivityAction`) are skipped.
223
+ * - **AI jobs the account started** (`orgs/{id}/aiJobs` by `createdBy`). The
224
+ * start only: a job's later updates are the platform's.
225
+ *
226
+ * Firebase's sign-in and token-refresh times, and the device "last seen"
227
+ * stamps, are never read: a silent re-sign-in moves them with nobody there.
228
+ */ export async function readRecentActivityAtMs(db, uid, sinceMs) {
229
+ var _instantMs;
230
+ const since = Timestamp.fromMillis(sinceMs);
231
+ const jobs = db.collectionGroup('aiJobs').where('createdBy', '==', uid).where('createdAt', '>=', since).orderBy('createdAt', 'desc').select('createdAt').limit(1).get();
232
+ const log = db.collectionGroup('activity').where('actorId', '==', uid).where('createdAt', '>=', since).orderBy('createdAt', 'desc').select('action', 'createdAt').limit(ACTIVITY_PAGE);
233
+ let newest = null;
234
+ let after;
235
+ // Newest first, so the first real entry is the newest; a page of nothing
236
+ // but platform entries reads the next one, a few pages at most.
237
+ for(let page = 0; page < ACTIVITY_PAGES && newest === null; page += 1){
238
+ var _instantMs1;
239
+ const entries = await (after ? log.startAfter(after) : log).get();
240
+ const real = entries.docs.find((doc)=>isRealActivityAction(doc.get('action')));
241
+ if (real) newest = (_instantMs1 = instantMs(real.get('createdAt'))) != null ? _instantMs1 : sinceMs;
242
+ if (entries.docs.length < ACTIVITY_PAGE) break;
243
+ after = entries.docs[entries.docs.length - 1];
244
+ }
245
+ const job = (await jobs).docs[0];
246
+ const started = job ? (_instantMs = instantMs(job.get('createdAt'))) != null ? _instantMs : sinceMs : null;
247
+ return newest === null ? started : started === null ? newest : Math.max(newest, started);
248
+ }
249
+ /** The eligible accounts, the counts of everyone left out, and each stage. */ export function selectAdAudience(accounts, internalDomains, now = Date.now()) {
116
250
  const excluded = Object.fromEntries(AD_AUDIENCE_EXCLUSIONS.map((reason)=>[
117
251
  reason,
118
252
  0
@@ -122,6 +256,10 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
122
256
  billing: 0,
123
257
  signup: 0
124
258
  };
259
+ const stages = Object.fromEntries(AD_AUDIENCE_STAGES.map((stage)=>[
260
+ stage,
261
+ []
262
+ ]));
125
263
  for (const account of accounts){
126
264
  const reason = adAudienceExclusion(account, {
127
265
  internalDomains
@@ -133,12 +271,14 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
133
271
  eligible.push(account);
134
272
  const { source } = resolveAccountCountry(account);
135
273
  if (source) countrySource[source] += 1;
274
+ for (const stage of adAudienceStages(account, now))stages[stage].push(account);
136
275
  }
137
276
  return {
138
277
  scanned: accounts.length,
139
278
  eligible,
140
279
  excluded,
141
- countrySource
280
+ countrySource,
281
+ stages
142
282
  };
143
283
  }
144
284
  /** The set one provider should hold. Two addresses may share a hash. */ export function providerHashes(provider, eligible) {
@@ -153,8 +293,11 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
153
293
  }
154
294
  /*==========================================
155
295
  * STATE
156
- *=========================================*/ /** What one provider was last sent. */ export async function readSentHashes(firestore, provider) {
157
- const chunks = await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(provider).collection(AD_AUDIENCE_MEMBERS_SUBCOLLECTION).get();
296
+ *=========================================*/ /**
297
+ * What one list was last sent: the provider's all-holders list, or with
298
+ * `stage` that stage's list, each from its own ledger.
299
+ */ export async function readSentHashes(firestore, provider, stage) {
300
+ const chunks = await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(provider, stage)).collection(AD_AUDIENCE_MEMBERS_SUBCOLLECTION).get();
158
301
  const hashes = [];
159
302
  for (const doc of chunks.docs){
160
303
  const held = doc.get('hashes');
@@ -162,8 +305,13 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
162
305
  }
163
306
  return hashes;
164
307
  }
165
- /** Replace what one provider was last sent, and its counts. */ async function writeSentState(firestore, provider, hashes, state) {
166
- const ref = firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(provider);
308
+ /** The fields that say which list a state document belongs to. */ const ledgerFields = (list)=>_extends({
309
+ provider: list.provider
310
+ }, list.stage ? {
311
+ stage: list.stage
312
+ } : {});
313
+ /** Replace what one list was last sent, and its counts. */ async function writeSentState(firestore, list, hashes, state) {
314
+ const ref = firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(list.provider, list.stage));
167
315
  const members = ref.collection(AD_AUDIENCE_MEMBERS_SUBCOLLECTION);
168
316
  const existing = await members.get();
169
317
  const runs = [];
@@ -179,19 +327,15 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
179
327
  for (const doc of existing.docs){
180
328
  if (!keep.has(doc.id)) await doc.ref.delete();
181
329
  }
182
- await ref.set(_extends({
183
- provider
184
- }, state, {
330
+ await ref.set(_extends({}, ledgerFields(list), state, {
185
331
  members: hashes.length,
186
332
  chunks: runs.length
187
333
  }), {
188
334
  merge: true
189
335
  });
190
336
  }
191
- /** The counts of a run on the provider's state document, members untouched. */ async function writeRunState(firestore, provider, state) {
192
- await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(provider).set(_extends({
193
- provider
194
- }, state), {
337
+ /** The counts of a run on the list's state document, members untouched. */ async function writeRunState(firestore, list, state) {
338
+ await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(list.provider, list.stage)).set(_extends({}, ledgerFields(list), state), {
195
339
  merge: true
196
340
  });
197
341
  }
@@ -246,109 +390,129 @@ async function sendLinkedIn(fetcher, config, diff) {
246
390
  }
247
391
  }
248
392
  }
393
+ /**
394
+ * One list on one provider: read what it was last sent, diff, and — unless
395
+ * this run may not send — send and record. The all-holders list and every
396
+ * stage list go through exactly this, each against its own ledger.
397
+ */ async function syncList(options, log, list, listConfig, hashes, /** Added to the clock for Meta's session id, which must be unique per run and audience. */ sessionOffset) {
398
+ const { provider, stage } = list;
399
+ const name = stage ? `${provider} ${stage}` : provider;
400
+ const config = listConfig[provider];
401
+ const base = {
402
+ eligible: hashes.length,
403
+ previous: 0,
404
+ toAdd: 0,
405
+ toRemove: 0
406
+ };
407
+ if (!config) {
408
+ // A stage list nobody set up is the normal case; its line is the stage's.
409
+ if (!stage) log(`[ad-audience-sync] ${provider} skipped — set ${listConfig.missing[provider].join(', ')}`);
410
+ return _extends({}, base, {
411
+ outcome: 'unconfigured',
412
+ missing: listConfig.missing[provider]
413
+ });
414
+ }
415
+ const previous = await readSentHashes(options.firestore, provider, stage);
416
+ const diff = diffAudience(previous, hashes);
417
+ const counts = _extends({}, base, {
418
+ previous: previous.length,
419
+ toAdd: diff.toAdd.length,
420
+ toRemove: diff.toRemove.length
421
+ });
422
+ const outcome = (value, extra = {})=>{
423
+ log(`[ad-audience-sync] ${name} ${value}: eligible ${counts.eligible}, ` + `previous ${counts.previous}, add ${counts.toAdd}, remove ${counts.toRemove}` + (extra.error ? ` — ${extra.error}` : ''));
424
+ return _extends({}, counts, {
425
+ outcome: value
426
+ }, extra);
427
+ };
428
+ if (options.dryRun) return outcome('dry-run');
429
+ if (!options.flagOn) return outcome('flag-off');
430
+ const stamp = {
431
+ lastRunAtMs: options.now,
432
+ eligible: counts.eligible
433
+ };
434
+ if (!diff.toAdd.length && !diff.toRemove.length) {
435
+ await writeRunState(options.firestore, list, _extends({}, stamp, {
436
+ lastOutcome: 'unchanged'
437
+ }));
438
+ return outcome('unchanged');
439
+ }
440
+ // Meta replaces with what it is sent, and an empty replace is no replace.
441
+ if (!hashes.length) {
442
+ await writeRunState(options.firestore, list, _extends({}, stamp, {
443
+ lastOutcome: 'empty'
444
+ }));
445
+ return outcome('empty', {
446
+ error: 'no eligible account; the audience was left as it is'
447
+ });
448
+ }
449
+ if (!options.force && isSuspiciousShrink(previous.length, hashes.length)) {
450
+ await writeRunState(options.firestore, list, _extends({}, stamp, {
451
+ lastOutcome: 'held-shrink'
452
+ }));
453
+ return outcome('held-shrink', {
454
+ error: `the audience would shrink from ${previous.length} to ${hashes.length}; held until a run with force`
455
+ });
456
+ }
457
+ try {
458
+ if (provider === 'meta') {
459
+ await sendMeta(options.fetch, config, hashes, options.now + sessionOffset);
460
+ } else if (provider === 'google') {
461
+ await sendGoogle(options.fetch, config, diff);
462
+ } else {
463
+ await sendLinkedIn(options.fetch, config, diff);
464
+ }
465
+ } catch (error) {
466
+ // The stored set stays as it was: the next run retries the whole diff,
467
+ // and every provider takes a repeated add or remove without harm.
468
+ const message = error instanceof AudienceSendError ? error.message : `${provider} send failed`;
469
+ if (!(error instanceof AudienceSendError)) console.error(`[ad-audience-sync] ${name}`, error);
470
+ await writeRunState(options.firestore, list, _extends({}, stamp, {
471
+ lastOutcome: 'failed',
472
+ lastError: message
473
+ }));
474
+ return outcome('failed', {
475
+ error: message
476
+ });
477
+ }
478
+ await writeSentState(options.firestore, list, hashes, _extends({}, stamp, {
479
+ lastOutcome: 'sent',
480
+ lastSentAtMs: options.now,
481
+ lastAdded: counts.toAdd,
482
+ lastRemoved: counts.toRemove,
483
+ lastError: null
484
+ }));
485
+ return outcome('sent');
486
+ }
249
487
  export async function runAdAudienceSync(options) {
250
488
  var _options_log;
251
489
  const log = (_options_log = options.log) != null ? _options_log : (line)=>console.info(line);
252
- const selection = selectAdAudience(options.accounts, options.internalDomains);
490
+ const selection = selectAdAudience(options.accounts, options.internalDomains, options.now);
253
491
  const providers = {};
254
492
  for (const provider of AD_AUDIENCE_PROVIDERS){
255
- const hashes = providerHashes(provider, selection.eligible);
256
- const config = options.config[provider];
257
- const base = {
258
- eligible: hashes.length,
259
- previous: 0,
260
- toAdd: 0,
261
- toRemove: 0
262
- };
263
- if (!config) {
264
- providers[provider] = _extends({}, base, {
265
- outcome: 'unconfigured',
266
- missing: options.config.missing[provider]
267
- });
268
- log(`[ad-audience-sync] ${provider} skipped — set ${options.config.missing[provider].join(', ')}`);
269
- continue;
270
- }
271
- const previous = await readSentHashes(options.firestore, provider);
272
- const diff = diffAudience(previous, hashes);
273
- const counts = _extends({}, base, {
274
- previous: previous.length,
275
- toAdd: diff.toAdd.length,
276
- toRemove: diff.toRemove.length
277
- });
278
- const outcome = (value, extra = {})=>{
279
- providers[provider] = _extends({}, counts, {
280
- outcome: value
281
- }, extra);
282
- log(`[ad-audience-sync] ${provider} ${value}: eligible ${counts.eligible}, ` + `previous ${counts.previous}, add ${counts.toAdd}, remove ${counts.toRemove}` + (extra.error ? ` — ${extra.error}` : ''));
283
- };
284
- if (options.dryRun) {
285
- outcome('dry-run');
286
- continue;
287
- }
288
- if (!options.flagOn) {
289
- outcome('flag-off');
290
- continue;
493
+ providers[provider] = await syncList(options, log, {
494
+ provider,
495
+ stage: null
496
+ }, options.config, providerHashes(provider, selection.eligible), 0);
497
+ }
498
+ // The stage lists: each a subset of the same eligible audience, each with
499
+ // its own ledger, diff and shrink guard. Each failing alone, like a provider.
500
+ const stages = {};
501
+ for (const [index, stage] of AD_AUDIENCE_STAGES.entries()){
502
+ const members = selection.stages[stage];
503
+ const stageProviders = {};
504
+ for (const provider of AD_AUDIENCE_PROVIDERS){
505
+ stageProviders[provider] = await syncList(options, log, {
506
+ provider,
507
+ stage
508
+ }, options.config.stages[stage], providerHashes(provider, members), index + 1);
291
509
  }
292
- const stamp = {
293
- lastRunAtMs: options.now,
294
- eligible: counts.eligible
510
+ const unset = AD_AUDIENCE_PROVIDERS.filter((provider)=>stageProviders[provider].outcome === 'unconfigured');
511
+ log(`[ad-audience-sync] stage ${stage}: eligible ${members.length}` + (unset.length ? `; no list set on ${unset.join(', ')}` : ''));
512
+ stages[stage] = {
513
+ eligible: members.length,
514
+ providers: stageProviders
295
515
  };
296
- if (!diff.toAdd.length && !diff.toRemove.length) {
297
- await writeRunState(options.firestore, provider, _extends({}, stamp, {
298
- lastOutcome: 'unchanged'
299
- }));
300
- outcome('unchanged');
301
- continue;
302
- }
303
- // Meta replaces with what it is sent, and an empty replace is no replace.
304
- if (!hashes.length) {
305
- await writeRunState(options.firestore, provider, _extends({}, stamp, {
306
- lastOutcome: 'empty'
307
- }));
308
- outcome('empty', {
309
- error: 'no eligible account; the audience was left as it is'
310
- });
311
- continue;
312
- }
313
- if (!options.force && isSuspiciousShrink(previous.length, hashes.length)) {
314
- await writeRunState(options.firestore, provider, _extends({}, stamp, {
315
- lastOutcome: 'held-shrink'
316
- }));
317
- outcome('held-shrink', {
318
- error: `the audience would shrink from ${previous.length} to ${hashes.length}; held until a run with force`
319
- });
320
- continue;
321
- }
322
- try {
323
- if (provider === 'meta') {
324
- await sendMeta(options.fetch, config, hashes, options.now);
325
- } else if (provider === 'google') {
326
- await sendGoogle(options.fetch, config, diff);
327
- } else {
328
- await sendLinkedIn(options.fetch, config, diff);
329
- }
330
- } catch (error) {
331
- // The stored set stays as it was: the next run retries the whole diff,
332
- // and every provider takes a repeated add or remove without harm.
333
- const message = error instanceof AudienceSendError ? error.message : `${provider} send failed`;
334
- if (!(error instanceof AudienceSendError)) console.error(`[ad-audience-sync] ${provider}`, error);
335
- await writeRunState(options.firestore, provider, _extends({}, stamp, {
336
- lastOutcome: 'failed',
337
- lastError: message
338
- }));
339
- outcome('failed', {
340
- error: message
341
- });
342
- continue;
343
- }
344
- await writeSentState(options.firestore, provider, hashes, _extends({}, stamp, {
345
- lastOutcome: 'sent',
346
- lastSentAtMs: options.now,
347
- lastAdded: counts.toAdd,
348
- lastRemoved: counts.toRemove,
349
- lastError: null
350
- }));
351
- outcome('sent');
352
516
  }
353
517
  return {
354
518
  dryRun: options.dryRun,
@@ -357,7 +521,8 @@ export async function runAdAudienceSync(options) {
357
521
  eligible: selection.eligible.length,
358
522
  excluded: selection.excluded,
359
523
  countrySource: selection.countrySource,
360
- providers
524
+ providers,
525
+ stages
361
526
  };
362
527
  }
363
528