@aglyn/tenant-data-admin 1.0.0-beta.284 → 1.0.0-beta.285
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/lib/server/ad-audience-sync-run.d.ts +63 -6
- package/src/lib/server/ad-audience-sync-run.js +289 -124
- package/src/lib/server/ad-audience-sync-run.js.map +1 -1
- package/src/lib/server/ad-audience-sync.d.ts +115 -2
- package/src/lib/server/ad-audience-sync.js +203 -25
- package/src/lib/server/ad-audience-sync.js.map +1 -1
|
@@ -17,8 +17,9 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
|
17
17
|
*/ import { AD_AUDIENCE_OPT_OUT_FIELD, readAdAudienceOptOut } from "@aglyn/aglyn/app-utils/ad-audience";
|
|
18
18
|
import { ACCOUNT_ACQUISITION_FIELD } from "@aglyn/aglyn/app-utils/account-acquisition";
|
|
19
19
|
import { isAccountBanLockdownReason, LOCKDOWNS_COLLECTION } from "@aglyn/aglyn/app-utils/lockdown";
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
20
|
+
import { resolveEffectivePlan } from "@aglyn/aglyn/app-utils/plan-entitlements";
|
|
21
|
+
import { FieldPath, Timestamp } from "firebase-admin/firestore";
|
|
22
|
+
import { AD_AUDIENCE_EXCLUSIONS, AD_AUDIENCE_LAPSE_MS, AD_AUDIENCE_PROVIDERS, AD_AUDIENCE_STAGES, adAudienceExclusion, adAudienceStages, buildGoogleMemberRequests, buildGoogleTokenRequest, buildLinkedInUserRequests, buildMetaReplaceRequests, diffAudience, hashAudienceEmail, instantMs, isOwnPublishedScreen, isRealActivityAction, isSuspiciousShrink, linkedInRefusedCount, resolveAccountCountry } from "./ad-audience-sync.js";
|
|
22
23
|
/**
|
|
23
24
|
* THE ACCOUNT-HOLDER AD AUDIENCE (AGL-3780) — reading, sending, recording.
|
|
24
25
|
*
|
|
@@ -35,11 +36,24 @@ import { AD_AUDIENCE_EXCLUSIONS, AD_AUDIENCE_PROVIDERS, adAudienceExclusion, bui
|
|
|
35
36
|
* the only way to take it out of a Google list or a LinkedIn segment is the
|
|
36
37
|
* hash this sync sent it under. Admin-SDK only; no client rule opens either.
|
|
37
38
|
* No address is stored, and neither addresses nor hashes are ever logged.
|
|
39
|
+
*
|
|
40
|
+
* A STAGE list (AGL-4072) keeps the same two things under its own document,
|
|
41
|
+
* `adAudienceSync/{provider}--{stage}`, so each stage has its own ledger, its
|
|
42
|
+
* own diff and its own shrink guard, and nothing one list does can touch what
|
|
43
|
+
* another was sent.
|
|
38
44
|
*/ export const AD_AUDIENCE_SYNC_COLLECTION = 'adAudienceSync';
|
|
39
45
|
export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
40
46
|
/** Hashes per member chunk: 5,000 × 64 bytes stays far under a document's 1 MiB. */ const MEMBER_CHUNK = 5000;
|
|
41
47
|
/** Auth pages walked per run (1,000 accounts each). */ const MAX_AUTH_PAGES = 500;
|
|
42
|
-
/**
|
|
48
|
+
/** Page documents read per page of the published-page scan. */ const SCREEN_PAGE = 5000;
|
|
49
|
+
/** Change-log entries read per page while looking for one the person made. */ const ACTIVITY_PAGE = 25;
|
|
50
|
+
/** Pages of change log read per account before it is called quiet. */ const ACTIVITY_PAGES = 4;
|
|
51
|
+
/** Accounts whose recent activity is read at once. */ const ACTIVITY_CONCURRENCY = 20;
|
|
52
|
+
/** The ledger document of one list: the provider's own, or `{provider}--{stage}`. */ export function adAudienceLedgerId(provider, stage) {
|
|
53
|
+
return stage ? `${provider}--${stage}` : provider;
|
|
54
|
+
}
|
|
55
|
+
/** Every account, with the facts eligibility and the stage lists read. */ export async function readAdAudienceAccounts(deps) {
|
|
56
|
+
var _deps_recentActivity, _deps_internalDomains;
|
|
43
57
|
const db = deps.firestore;
|
|
44
58
|
// Profiles: the opt-out and the sign-up country, in one query.
|
|
45
59
|
const profiles = await db.collection('users').select(AD_AUDIENCE_OPT_OUT_FIELD, `${ACCOUNT_ACQUISITION_FIELD}.geo.country`).get();
|
|
@@ -53,19 +67,42 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
53
67
|
const geo = (_data_ACCOUNT_ACQUISITION_FIELD = data[ACCOUNT_ACQUISITION_FIELD]) == null ? void 0 : _data_ACCOUNT_ACQUISITION_FIELD.geo;
|
|
54
68
|
if (typeof (geo == null ? void 0 : geo.country) === 'string') signupCountry.set(doc.id, geo.country);
|
|
55
69
|
}
|
|
56
|
-
//
|
|
57
|
-
|
|
70
|
+
// The workspaces each account owns, in one query: their billing countries,
|
|
71
|
+
// whether any is on a paid plan, and the sites they hold. The plan is
|
|
72
|
+
// `resolveEffectivePlan`'s, the one every entitlement reads — a canceled
|
|
73
|
+
// subscription is Free, a staff comp is the plan it grants — and it needs
|
|
74
|
+
// only the org document's own fields (the derived `billingStatus` included).
|
|
75
|
+
const orgs = await db.collection('orgs').select('ownerUid', 'contact.address.country', 'plan', 'billingStatus', 'subscription.status', 'entitlements.planComp', 'hosts').get();
|
|
58
76
|
const billing = new Map();
|
|
77
|
+
const paidOwners = new Set();
|
|
78
|
+
const hostOwner = new Map();
|
|
59
79
|
for (const doc of orgs.docs){
|
|
60
|
-
var
|
|
80
|
+
var _doc_data1;
|
|
61
81
|
const owner = doc.get('ownerUid');
|
|
82
|
+
if (typeof owner !== 'string' || !owner) continue;
|
|
62
83
|
const country = doc.get('contact.address.country');
|
|
63
|
-
if (typeof
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
84
|
+
if (typeof country === 'string' && country) {
|
|
85
|
+
var _billing_get;
|
|
86
|
+
billing.set(owner, [
|
|
87
|
+
...(_billing_get = billing.get(owner)) != null ? _billing_get : [],
|
|
88
|
+
country
|
|
89
|
+
]);
|
|
90
|
+
}
|
|
91
|
+
const data = (_doc_data1 = doc.data()) != null ? _doc_data1 : {};
|
|
92
|
+
if (resolveEffectivePlan(data) !== 'free') paidOwners.add(owner);
|
|
93
|
+
const hosts = data['hosts'];
|
|
94
|
+
if (hosts && typeof hosts === 'object') {
|
|
95
|
+
for (const hostId of Object.keys(hosts))hostOwner.set(hostId, owner);
|
|
96
|
+
}
|
|
68
97
|
}
|
|
98
|
+
// The stage facts never hold up the all-holders list: a read that fails
|
|
99
|
+
// leaves the accounts in no stage tonight, which every stage list's shrink
|
|
100
|
+
// hold (or its refusal to send an empty replace) keeps from reaching a
|
|
101
|
+
// provider, and the main list is synced as ever.
|
|
102
|
+
const publishedOwners = await readOwnPublishedOwners(db, hostOwner, deps.now).catch((error)=>{
|
|
103
|
+
console.error('[ad-audience-sync] the published-page read failed; no stage tonight', error);
|
|
104
|
+
return null;
|
|
105
|
+
});
|
|
69
106
|
// Account locks: `lockdowns/user--{uid}`, by document id range.
|
|
70
107
|
const locks = await db.collection(LOCKDOWNS_COLLECTION).where(FieldPath.documentId(), '>=', 'user--').where(FieldPath.documentId(), '<', 'user-.').get();
|
|
71
108
|
const locked = new Set();
|
|
@@ -83,8 +120,8 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
83
120
|
const listed = await deps.listUserPage(1000, pageToken);
|
|
84
121
|
for (const { record } of listed.users){
|
|
85
122
|
var _record_email, _billing_get1, _signupCountry_get;
|
|
86
|
-
var _record_customClaims;
|
|
87
|
-
accounts.push({
|
|
123
|
+
var _record_customClaims, _record_metadata;
|
|
124
|
+
accounts.push(_extends({
|
|
88
125
|
uid: record.uid,
|
|
89
126
|
email: (_record_email = record.email) != null ? _record_email : null,
|
|
90
127
|
emailVerified: Boolean(record.emailVerified),
|
|
@@ -95,7 +132,14 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
95
132
|
banned: false,
|
|
96
133
|
billingCountries: (_billing_get1 = billing.get(record.uid)) != null ? _billing_get1 : [],
|
|
97
134
|
signupCountry: (_signupCountry_get = signupCountry.get(record.uid)) != null ? _signupCountry_get : null
|
|
98
|
-
}
|
|
135
|
+
}, publishedOwners ? {
|
|
136
|
+
stageFacts: {
|
|
137
|
+
createdAtMs: instantMs((_record_metadata = record.metadata) == null ? void 0 : _record_metadata.creationTime),
|
|
138
|
+
ownsPaidWorkspace: paidOwners.has(record.uid),
|
|
139
|
+
ownsPublishedSite: publishedOwners.has(record.uid),
|
|
140
|
+
lastActivityAtMs: null
|
|
141
|
+
}
|
|
142
|
+
} : {}));
|
|
99
143
|
}
|
|
100
144
|
pageToken = (_listed_nextPageToken = listed.nextPageToken) != null ? _listed_nextPageToken : undefined;
|
|
101
145
|
if (!pageToken) break;
|
|
@@ -108,11 +152,101 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
108
152
|
for (const account of accounts){
|
|
109
153
|
if (account.email && banned.has(account.email.trim().toLowerCase())) account.banned = true;
|
|
110
154
|
}
|
|
155
|
+
// Recent activity, for the only accounts it can change anything for: those
|
|
156
|
+
// in the audience and old enough to lapse. Everyone else's change log is
|
|
157
|
+
// never read, which keeps the nightly cost to the accounts that matter.
|
|
158
|
+
const since = deps.now - AD_AUDIENCE_LAPSE_MS;
|
|
159
|
+
const recentActivity = (_deps_recentActivity = deps.recentActivity) != null ? _deps_recentActivity : (uid, sinceMs)=>readRecentActivityAtMs(db, uid, sinceMs);
|
|
160
|
+
const internalDomains = (_deps_internalDomains = deps.internalDomains) != null ? _deps_internalDomains : [];
|
|
161
|
+
const candidates = accounts.filter((account)=>{
|
|
162
|
+
var _account_stageFacts;
|
|
163
|
+
const created = (_account_stageFacts = account.stageFacts) == null ? void 0 : _account_stageFacts.createdAtMs;
|
|
164
|
+
return typeof created === 'number' && created < since && !adAudienceExclusion(account, {
|
|
165
|
+
internalDomains
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
let activityFailures = 0;
|
|
169
|
+
for(let index = 0; index < candidates.length; index += ACTIVITY_CONCURRENCY){
|
|
170
|
+
await Promise.all(candidates.slice(index, index + ACTIVITY_CONCURRENCY).map(async (account)=>{
|
|
171
|
+
try {
|
|
172
|
+
const at = await recentActivity(account.uid, since);
|
|
173
|
+
if (account.stageFacts) account.stageFacts.lastActivityAtMs = at;
|
|
174
|
+
} catch (unused) {
|
|
175
|
+
// Unknown is not quiet: out of every stage tonight, never lapsed by mistake.
|
|
176
|
+
delete account.stageFacts;
|
|
177
|
+
activityFailures += 1;
|
|
178
|
+
}
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
181
|
+
if (activityFailures) {
|
|
182
|
+
console.error(`[ad-audience-sync] recent activity unread for ${activityFailures} accounts; in no stage tonight`);
|
|
183
|
+
}
|
|
111
184
|
return {
|
|
112
185
|
accounts
|
|
113
186
|
};
|
|
114
187
|
}
|
|
115
|
-
/**
|
|
188
|
+
/**
|
|
189
|
+
* The owners of a site with a page of their own published, from one scan of
|
|
190
|
+
* every page document (`hosts/{hostId}/screens/{id}`), paged by path and
|
|
191
|
+
* reading only the fields the test needs. One query instead of one per site;
|
|
192
|
+
* a page whose site no workspace holds counts for nobody.
|
|
193
|
+
*/ async function readOwnPublishedOwners(db, hostOwner, now) {
|
|
194
|
+
const owners = new Set();
|
|
195
|
+
if (!hostOwner.size) return owners;
|
|
196
|
+
const base = db.collectionGroup('screens').orderBy(FieldPath.documentId()).select('publishedAt', 'deletedAt', 'createdAt', 'updatedAt', 'createdBy').limit(SCREEN_PAGE);
|
|
197
|
+
let after;
|
|
198
|
+
for(;;){
|
|
199
|
+
const page = await (after ? base.startAfter(after) : base).get();
|
|
200
|
+
for (const doc of page.docs){
|
|
201
|
+
var _doc_data;
|
|
202
|
+
// `hosts/{hostId}/screens/{screenId}`; any other `screens` is not a site's.
|
|
203
|
+
const [root, hostId] = doc.ref.path.split('/');
|
|
204
|
+
const owner = root === 'hosts' ? hostOwner.get(hostId) : undefined;
|
|
205
|
+
if (!owner || owners.has(owner)) continue;
|
|
206
|
+
if (isOwnPublishedScreen((_doc_data = doc.data()) != null ? _doc_data : {}, now)) owners.add(owner);
|
|
207
|
+
}
|
|
208
|
+
if (page.docs.length < SCREEN_PAGE) break;
|
|
209
|
+
after = page.docs[page.docs.length - 1];
|
|
210
|
+
}
|
|
211
|
+
return owners;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* The newest thing one account DID since `sinceMs`, or null when nothing was
|
|
215
|
+
* recorded. Two signals, both read through collection-group indexes that
|
|
216
|
+
* already exist for the staff activity feed and the AI job list:
|
|
217
|
+
*
|
|
218
|
+
* - **Change-log entries** under the account's id, on any site's or
|
|
219
|
+
* workspace's log (`hosts/{id}/activity`, `orgs/{id}/activity`): page
|
|
220
|
+
* edits, publishes, form and CRM actions, settings. Entries the platform
|
|
221
|
+
* writes under the person's id without them acting
|
|
222
|
+
* (`isRealActivityAction`) are skipped.
|
|
223
|
+
* - **AI jobs the account started** (`orgs/{id}/aiJobs` by `createdBy`). The
|
|
224
|
+
* start only: a job's later updates are the platform's.
|
|
225
|
+
*
|
|
226
|
+
* Firebase's sign-in and token-refresh times, and the device "last seen"
|
|
227
|
+
* stamps, are never read: a silent re-sign-in moves them with nobody there.
|
|
228
|
+
*/ export async function readRecentActivityAtMs(db, uid, sinceMs) {
|
|
229
|
+
var _instantMs;
|
|
230
|
+
const since = Timestamp.fromMillis(sinceMs);
|
|
231
|
+
const jobs = db.collectionGroup('aiJobs').where('createdBy', '==', uid).where('createdAt', '>=', since).orderBy('createdAt', 'desc').select('createdAt').limit(1).get();
|
|
232
|
+
const log = db.collectionGroup('activity').where('actorId', '==', uid).where('createdAt', '>=', since).orderBy('createdAt', 'desc').select('action', 'createdAt').limit(ACTIVITY_PAGE);
|
|
233
|
+
let newest = null;
|
|
234
|
+
let after;
|
|
235
|
+
// Newest first, so the first real entry is the newest; a page of nothing
|
|
236
|
+
// but platform entries reads the next one, a few pages at most.
|
|
237
|
+
for(let page = 0; page < ACTIVITY_PAGES && newest === null; page += 1){
|
|
238
|
+
var _instantMs1;
|
|
239
|
+
const entries = await (after ? log.startAfter(after) : log).get();
|
|
240
|
+
const real = entries.docs.find((doc)=>isRealActivityAction(doc.get('action')));
|
|
241
|
+
if (real) newest = (_instantMs1 = instantMs(real.get('createdAt'))) != null ? _instantMs1 : sinceMs;
|
|
242
|
+
if (entries.docs.length < ACTIVITY_PAGE) break;
|
|
243
|
+
after = entries.docs[entries.docs.length - 1];
|
|
244
|
+
}
|
|
245
|
+
const job = (await jobs).docs[0];
|
|
246
|
+
const started = job ? (_instantMs = instantMs(job.get('createdAt'))) != null ? _instantMs : sinceMs : null;
|
|
247
|
+
return newest === null ? started : started === null ? newest : Math.max(newest, started);
|
|
248
|
+
}
|
|
249
|
+
/** The eligible accounts, the counts of everyone left out, and each stage. */ export function selectAdAudience(accounts, internalDomains, now = Date.now()) {
|
|
116
250
|
const excluded = Object.fromEntries(AD_AUDIENCE_EXCLUSIONS.map((reason)=>[
|
|
117
251
|
reason,
|
|
118
252
|
0
|
|
@@ -122,6 +256,10 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
122
256
|
billing: 0,
|
|
123
257
|
signup: 0
|
|
124
258
|
};
|
|
259
|
+
const stages = Object.fromEntries(AD_AUDIENCE_STAGES.map((stage)=>[
|
|
260
|
+
stage,
|
|
261
|
+
[]
|
|
262
|
+
]));
|
|
125
263
|
for (const account of accounts){
|
|
126
264
|
const reason = adAudienceExclusion(account, {
|
|
127
265
|
internalDomains
|
|
@@ -133,12 +271,14 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
133
271
|
eligible.push(account);
|
|
134
272
|
const { source } = resolveAccountCountry(account);
|
|
135
273
|
if (source) countrySource[source] += 1;
|
|
274
|
+
for (const stage of adAudienceStages(account, now))stages[stage].push(account);
|
|
136
275
|
}
|
|
137
276
|
return {
|
|
138
277
|
scanned: accounts.length,
|
|
139
278
|
eligible,
|
|
140
279
|
excluded,
|
|
141
|
-
countrySource
|
|
280
|
+
countrySource,
|
|
281
|
+
stages
|
|
142
282
|
};
|
|
143
283
|
}
|
|
144
284
|
/** The set one provider should hold. Two addresses may share a hash. */ export function providerHashes(provider, eligible) {
|
|
@@ -153,8 +293,11 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
153
293
|
}
|
|
154
294
|
/*==========================================
|
|
155
295
|
* STATE
|
|
156
|
-
*=========================================*/ /**
|
|
157
|
-
|
|
296
|
+
*=========================================*/ /**
|
|
297
|
+
* What one list was last sent: the provider's all-holders list, or with
|
|
298
|
+
* `stage` that stage's list, each from its own ledger.
|
|
299
|
+
*/ export async function readSentHashes(firestore, provider, stage) {
|
|
300
|
+
const chunks = await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(provider, stage)).collection(AD_AUDIENCE_MEMBERS_SUBCOLLECTION).get();
|
|
158
301
|
const hashes = [];
|
|
159
302
|
for (const doc of chunks.docs){
|
|
160
303
|
const held = doc.get('hashes');
|
|
@@ -162,8 +305,13 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
162
305
|
}
|
|
163
306
|
return hashes;
|
|
164
307
|
}
|
|
165
|
-
/**
|
|
166
|
-
|
|
308
|
+
/** The fields that say which list a state document belongs to. */ const ledgerFields = (list)=>_extends({
|
|
309
|
+
provider: list.provider
|
|
310
|
+
}, list.stage ? {
|
|
311
|
+
stage: list.stage
|
|
312
|
+
} : {});
|
|
313
|
+
/** Replace what one list was last sent, and its counts. */ async function writeSentState(firestore, list, hashes, state) {
|
|
314
|
+
const ref = firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(list.provider, list.stage));
|
|
167
315
|
const members = ref.collection(AD_AUDIENCE_MEMBERS_SUBCOLLECTION);
|
|
168
316
|
const existing = await members.get();
|
|
169
317
|
const runs = [];
|
|
@@ -179,19 +327,15 @@ export const AD_AUDIENCE_MEMBERS_SUBCOLLECTION = 'members';
|
|
|
179
327
|
for (const doc of existing.docs){
|
|
180
328
|
if (!keep.has(doc.id)) await doc.ref.delete();
|
|
181
329
|
}
|
|
182
|
-
await ref.set(_extends({
|
|
183
|
-
provider
|
|
184
|
-
}, state, {
|
|
330
|
+
await ref.set(_extends({}, ledgerFields(list), state, {
|
|
185
331
|
members: hashes.length,
|
|
186
332
|
chunks: runs.length
|
|
187
333
|
}), {
|
|
188
334
|
merge: true
|
|
189
335
|
});
|
|
190
336
|
}
|
|
191
|
-
/** The counts of a run on the
|
|
192
|
-
await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(provider).set(_extends({
|
|
193
|
-
provider
|
|
194
|
-
}, state), {
|
|
337
|
+
/** The counts of a run on the list's state document, members untouched. */ async function writeRunState(firestore, list, state) {
|
|
338
|
+
await firestore.collection(AD_AUDIENCE_SYNC_COLLECTION).doc(adAudienceLedgerId(list.provider, list.stage)).set(_extends({}, ledgerFields(list), state), {
|
|
195
339
|
merge: true
|
|
196
340
|
});
|
|
197
341
|
}
|
|
@@ -246,109 +390,129 @@ async function sendLinkedIn(fetcher, config, diff) {
|
|
|
246
390
|
}
|
|
247
391
|
}
|
|
248
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* One list on one provider: read what it was last sent, diff, and — unless
|
|
395
|
+
* this run may not send — send and record. The all-holders list and every
|
|
396
|
+
* stage list go through exactly this, each against its own ledger.
|
|
397
|
+
*/ async function syncList(options, log, list, listConfig, hashes, /** Added to the clock for Meta's session id, which must be unique per run and audience. */ sessionOffset) {
|
|
398
|
+
const { provider, stage } = list;
|
|
399
|
+
const name = stage ? `${provider} ${stage}` : provider;
|
|
400
|
+
const config = listConfig[provider];
|
|
401
|
+
const base = {
|
|
402
|
+
eligible: hashes.length,
|
|
403
|
+
previous: 0,
|
|
404
|
+
toAdd: 0,
|
|
405
|
+
toRemove: 0
|
|
406
|
+
};
|
|
407
|
+
if (!config) {
|
|
408
|
+
// A stage list nobody set up is the normal case; its line is the stage's.
|
|
409
|
+
if (!stage) log(`[ad-audience-sync] ${provider} skipped — set ${listConfig.missing[provider].join(', ')}`);
|
|
410
|
+
return _extends({}, base, {
|
|
411
|
+
outcome: 'unconfigured',
|
|
412
|
+
missing: listConfig.missing[provider]
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
const previous = await readSentHashes(options.firestore, provider, stage);
|
|
416
|
+
const diff = diffAudience(previous, hashes);
|
|
417
|
+
const counts = _extends({}, base, {
|
|
418
|
+
previous: previous.length,
|
|
419
|
+
toAdd: diff.toAdd.length,
|
|
420
|
+
toRemove: diff.toRemove.length
|
|
421
|
+
});
|
|
422
|
+
const outcome = (value, extra = {})=>{
|
|
423
|
+
log(`[ad-audience-sync] ${name} ${value}: eligible ${counts.eligible}, ` + `previous ${counts.previous}, add ${counts.toAdd}, remove ${counts.toRemove}` + (extra.error ? ` — ${extra.error}` : ''));
|
|
424
|
+
return _extends({}, counts, {
|
|
425
|
+
outcome: value
|
|
426
|
+
}, extra);
|
|
427
|
+
};
|
|
428
|
+
if (options.dryRun) return outcome('dry-run');
|
|
429
|
+
if (!options.flagOn) return outcome('flag-off');
|
|
430
|
+
const stamp = {
|
|
431
|
+
lastRunAtMs: options.now,
|
|
432
|
+
eligible: counts.eligible
|
|
433
|
+
};
|
|
434
|
+
if (!diff.toAdd.length && !diff.toRemove.length) {
|
|
435
|
+
await writeRunState(options.firestore, list, _extends({}, stamp, {
|
|
436
|
+
lastOutcome: 'unchanged'
|
|
437
|
+
}));
|
|
438
|
+
return outcome('unchanged');
|
|
439
|
+
}
|
|
440
|
+
// Meta replaces with what it is sent, and an empty replace is no replace.
|
|
441
|
+
if (!hashes.length) {
|
|
442
|
+
await writeRunState(options.firestore, list, _extends({}, stamp, {
|
|
443
|
+
lastOutcome: 'empty'
|
|
444
|
+
}));
|
|
445
|
+
return outcome('empty', {
|
|
446
|
+
error: 'no eligible account; the audience was left as it is'
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
if (!options.force && isSuspiciousShrink(previous.length, hashes.length)) {
|
|
450
|
+
await writeRunState(options.firestore, list, _extends({}, stamp, {
|
|
451
|
+
lastOutcome: 'held-shrink'
|
|
452
|
+
}));
|
|
453
|
+
return outcome('held-shrink', {
|
|
454
|
+
error: `the audience would shrink from ${previous.length} to ${hashes.length}; held until a run with force`
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
try {
|
|
458
|
+
if (provider === 'meta') {
|
|
459
|
+
await sendMeta(options.fetch, config, hashes, options.now + sessionOffset);
|
|
460
|
+
} else if (provider === 'google') {
|
|
461
|
+
await sendGoogle(options.fetch, config, diff);
|
|
462
|
+
} else {
|
|
463
|
+
await sendLinkedIn(options.fetch, config, diff);
|
|
464
|
+
}
|
|
465
|
+
} catch (error) {
|
|
466
|
+
// The stored set stays as it was: the next run retries the whole diff,
|
|
467
|
+
// and every provider takes a repeated add or remove without harm.
|
|
468
|
+
const message = error instanceof AudienceSendError ? error.message : `${provider} send failed`;
|
|
469
|
+
if (!(error instanceof AudienceSendError)) console.error(`[ad-audience-sync] ${name}`, error);
|
|
470
|
+
await writeRunState(options.firestore, list, _extends({}, stamp, {
|
|
471
|
+
lastOutcome: 'failed',
|
|
472
|
+
lastError: message
|
|
473
|
+
}));
|
|
474
|
+
return outcome('failed', {
|
|
475
|
+
error: message
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
await writeSentState(options.firestore, list, hashes, _extends({}, stamp, {
|
|
479
|
+
lastOutcome: 'sent',
|
|
480
|
+
lastSentAtMs: options.now,
|
|
481
|
+
lastAdded: counts.toAdd,
|
|
482
|
+
lastRemoved: counts.toRemove,
|
|
483
|
+
lastError: null
|
|
484
|
+
}));
|
|
485
|
+
return outcome('sent');
|
|
486
|
+
}
|
|
249
487
|
export async function runAdAudienceSync(options) {
|
|
250
488
|
var _options_log;
|
|
251
489
|
const log = (_options_log = options.log) != null ? _options_log : (line)=>console.info(line);
|
|
252
|
-
const selection = selectAdAudience(options.accounts, options.internalDomains);
|
|
490
|
+
const selection = selectAdAudience(options.accounts, options.internalDomains, options.now);
|
|
253
491
|
const providers = {};
|
|
254
492
|
for (const provider of AD_AUDIENCE_PROVIDERS){
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
const previous = await readSentHashes(options.firestore, provider);
|
|
272
|
-
const diff = diffAudience(previous, hashes);
|
|
273
|
-
const counts = _extends({}, base, {
|
|
274
|
-
previous: previous.length,
|
|
275
|
-
toAdd: diff.toAdd.length,
|
|
276
|
-
toRemove: diff.toRemove.length
|
|
277
|
-
});
|
|
278
|
-
const outcome = (value, extra = {})=>{
|
|
279
|
-
providers[provider] = _extends({}, counts, {
|
|
280
|
-
outcome: value
|
|
281
|
-
}, extra);
|
|
282
|
-
log(`[ad-audience-sync] ${provider} ${value}: eligible ${counts.eligible}, ` + `previous ${counts.previous}, add ${counts.toAdd}, remove ${counts.toRemove}` + (extra.error ? ` — ${extra.error}` : ''));
|
|
283
|
-
};
|
|
284
|
-
if (options.dryRun) {
|
|
285
|
-
outcome('dry-run');
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
288
|
-
if (!options.flagOn) {
|
|
289
|
-
outcome('flag-off');
|
|
290
|
-
continue;
|
|
493
|
+
providers[provider] = await syncList(options, log, {
|
|
494
|
+
provider,
|
|
495
|
+
stage: null
|
|
496
|
+
}, options.config, providerHashes(provider, selection.eligible), 0);
|
|
497
|
+
}
|
|
498
|
+
// The stage lists: each a subset of the same eligible audience, each with
|
|
499
|
+
// its own ledger, diff and shrink guard. Each failing alone, like a provider.
|
|
500
|
+
const stages = {};
|
|
501
|
+
for (const [index, stage] of AD_AUDIENCE_STAGES.entries()){
|
|
502
|
+
const members = selection.stages[stage];
|
|
503
|
+
const stageProviders = {};
|
|
504
|
+
for (const provider of AD_AUDIENCE_PROVIDERS){
|
|
505
|
+
stageProviders[provider] = await syncList(options, log, {
|
|
506
|
+
provider,
|
|
507
|
+
stage
|
|
508
|
+
}, options.config.stages[stage], providerHashes(provider, members), index + 1);
|
|
291
509
|
}
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
510
|
+
const unset = AD_AUDIENCE_PROVIDERS.filter((provider)=>stageProviders[provider].outcome === 'unconfigured');
|
|
511
|
+
log(`[ad-audience-sync] stage ${stage}: eligible ${members.length}` + (unset.length ? `; no list set on ${unset.join(', ')}` : ''));
|
|
512
|
+
stages[stage] = {
|
|
513
|
+
eligible: members.length,
|
|
514
|
+
providers: stageProviders
|
|
295
515
|
};
|
|
296
|
-
if (!diff.toAdd.length && !diff.toRemove.length) {
|
|
297
|
-
await writeRunState(options.firestore, provider, _extends({}, stamp, {
|
|
298
|
-
lastOutcome: 'unchanged'
|
|
299
|
-
}));
|
|
300
|
-
outcome('unchanged');
|
|
301
|
-
continue;
|
|
302
|
-
}
|
|
303
|
-
// Meta replaces with what it is sent, and an empty replace is no replace.
|
|
304
|
-
if (!hashes.length) {
|
|
305
|
-
await writeRunState(options.firestore, provider, _extends({}, stamp, {
|
|
306
|
-
lastOutcome: 'empty'
|
|
307
|
-
}));
|
|
308
|
-
outcome('empty', {
|
|
309
|
-
error: 'no eligible account; the audience was left as it is'
|
|
310
|
-
});
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
if (!options.force && isSuspiciousShrink(previous.length, hashes.length)) {
|
|
314
|
-
await writeRunState(options.firestore, provider, _extends({}, stamp, {
|
|
315
|
-
lastOutcome: 'held-shrink'
|
|
316
|
-
}));
|
|
317
|
-
outcome('held-shrink', {
|
|
318
|
-
error: `the audience would shrink from ${previous.length} to ${hashes.length}; held until a run with force`
|
|
319
|
-
});
|
|
320
|
-
continue;
|
|
321
|
-
}
|
|
322
|
-
try {
|
|
323
|
-
if (provider === 'meta') {
|
|
324
|
-
await sendMeta(options.fetch, config, hashes, options.now);
|
|
325
|
-
} else if (provider === 'google') {
|
|
326
|
-
await sendGoogle(options.fetch, config, diff);
|
|
327
|
-
} else {
|
|
328
|
-
await sendLinkedIn(options.fetch, config, diff);
|
|
329
|
-
}
|
|
330
|
-
} catch (error) {
|
|
331
|
-
// The stored set stays as it was: the next run retries the whole diff,
|
|
332
|
-
// and every provider takes a repeated add or remove without harm.
|
|
333
|
-
const message = error instanceof AudienceSendError ? error.message : `${provider} send failed`;
|
|
334
|
-
if (!(error instanceof AudienceSendError)) console.error(`[ad-audience-sync] ${provider}`, error);
|
|
335
|
-
await writeRunState(options.firestore, provider, _extends({}, stamp, {
|
|
336
|
-
lastOutcome: 'failed',
|
|
337
|
-
lastError: message
|
|
338
|
-
}));
|
|
339
|
-
outcome('failed', {
|
|
340
|
-
error: message
|
|
341
|
-
});
|
|
342
|
-
continue;
|
|
343
|
-
}
|
|
344
|
-
await writeSentState(options.firestore, provider, hashes, _extends({}, stamp, {
|
|
345
|
-
lastOutcome: 'sent',
|
|
346
|
-
lastSentAtMs: options.now,
|
|
347
|
-
lastAdded: counts.toAdd,
|
|
348
|
-
lastRemoved: counts.toRemove,
|
|
349
|
-
lastError: null
|
|
350
|
-
}));
|
|
351
|
-
outcome('sent');
|
|
352
516
|
}
|
|
353
517
|
return {
|
|
354
518
|
dryRun: options.dryRun,
|
|
@@ -357,7 +521,8 @@ export async function runAdAudienceSync(options) {
|
|
|
357
521
|
eligible: selection.eligible.length,
|
|
358
522
|
excluded: selection.excluded,
|
|
359
523
|
countrySource: selection.countrySource,
|
|
360
|
-
providers
|
|
524
|
+
providers,
|
|
525
|
+
stages
|
|
361
526
|
};
|
|
362
527
|
}
|
|
363
528
|
|