@mpgd/game-services 0.33.1 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admob-ssv.js +6 -26
- package/dist/app-store-verifier-conformance.d.ts +37 -1
- package/dist/app-store-verifier-conformance.js +199 -1
- package/dist/app-store-verifier.d.ts +57 -4
- package/dist/app-store-verifier.js +106 -4
- package/dist/base64.d.ts +10 -0
- package/dist/base64.js +35 -0
- package/dist/google-play-publisher-client.js +24 -1
- package/dist/google-play-rtdn.d.ts +248 -0
- package/dist/google-play-rtdn.js +594 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/notifications/ait-functional-delivery.d.ts +65 -1
- package/dist/notifications/ait-functional-delivery.js +239 -5
- package/dist/notifications/bulk.d.ts +158 -0
- package/dist/notifications/bulk.js +229 -0
- package/dist/notifications/client.d.ts +19 -1
- package/dist/notifications/client.js +24 -2
- package/dist/notifications/copy.d.ts +87 -0
- package/dist/notifications/copy.js +70 -0
- package/dist/notifications/delivery-provider.d.ts +79 -0
- package/dist/notifications/index.d.ts +4 -1
- package/dist/notifications/index.js +4 -1
- package/dist/notifications/registry.js +8 -0
- package/dist/notifications/service.d.ts +32 -0
- package/dist/notifications/service.js +1017 -59
- package/dist/notifications/store.d.ts +6 -0
- package/dist/notifications/store.js +5 -3
- package/dist/notifications/text.d.ts +5 -1
- package/dist/notifications/text.js +11 -2
- package/package.json +9 -5
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import { assertNotificationConsentRecord, assertNotificationDeployment, assertNotificationEndpointRegistration, assertRecordNotificationNoticeInput, createNotificationNoticeIdempotencyKey, createNotificationReminderIdempotencyKey, evaluateNotificationDelivery, evaluateNotificationReminderConsent, fillNotificationTemplate, isNotificationTimestamp, isSameNotificationNotice, isValidNotificationTimeZone, latestNotificationConsent, listNotificationNotices, normalizeNotificationTimestamp, NotificationError, notificationKeyIssue, notificationNoticeCopyIssues, notificationNoticeSendFrom, notificationNoticeTopic, notificationProviderCarriesNotices, notificationReservedKeyPrefix, notificationSendFingerprint, notificationShellRenderedAdvertisingTargets, notificationTemplateDataIssue, toSeoulCalendarDate, } from '@mpgd/platform/notifications';
|
|
1
|
+
import { assertNotificationConsentRecord, assertNotificationDeployment, assertNotificationEndpointRegistration, assertRecordNotificationNoticeInput, createNotificationNoticeIdempotencyKey, createNotificationReminderIdempotencyKey, evaluateNotificationDelivery, evaluateNotificationReminderConsent, fillNotificationTemplate, isNotificationTimestamp, isSameNotificationNotice, isValidNotificationTimeZone, latestNotificationConsent, listNotificationNotices, normalizeNotificationTimestamp, notificationContentSource, NotificationError, notificationKeyIssue, notificationNoticeCopyIssues, notificationNoticeSendFrom, notificationNoticeTopic, notificationProviderCarriesNotices, notificationReservedKeyPrefix, notificationSendFingerprint, notificationShellRenderedAdvertisingTargets, notificationTemplateDataIssue, toSeoulCalendarDate, } from '@mpgd/platform/notifications';
|
|
2
|
+
import { createNotificationBatchBarrier, createNotificationLimiter, isInsideNotificationHoldWindow, parseNotificationBulkRequest, } from './bulk.js';
|
|
3
|
+
import { resolveCatalogNotificationCopy, resolveNotificationCopy, resolveNotificationCopyByLocale, } from './copy.js';
|
|
2
4
|
import { fcmInstallationIdRefusal, notificationExpiredProviderCode, } from './delivery-provider.js';
|
|
3
5
|
import { notificationHistoryRetentionDays, } from './registry.js';
|
|
4
6
|
import { bestNotificationOutcome, formatNotificationInstant, isNotificationRowAbandoned, isNotificationRowDue, notificationAmbiguousReplayHours, notificationSendOutcome, } from './store.js';
|
|
@@ -118,6 +120,20 @@ export class NotificationConsentRateLimitError extends NotificationError {
|
|
|
118
120
|
class NotificationSendInputError extends Error {
|
|
119
121
|
name = 'NotificationSendInputError';
|
|
120
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* The copy resolver could not answer. Retryable, like a host that cannot
|
|
125
|
+
* answer, but a send's first check fails as a whole on it: copy is resolved
|
|
126
|
+
* per locale, so skipping only the recipients of one locale would store a
|
|
127
|
+
* send that never reaches them.
|
|
128
|
+
*/
|
|
129
|
+
class NotificationCopyUnavailableError extends NotificationError {
|
|
130
|
+
constructor(topic, options) {
|
|
131
|
+
super('NOTIFICATIONS_TEMPORARILY_UNAVAILABLE', {
|
|
132
|
+
message: `The copy resolver could not answer for topic ${topic}.`,
|
|
133
|
+
cause: options.cause,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
121
137
|
/**
|
|
122
138
|
* The provider accepted or may have sent, but storing its answer failed. It
|
|
123
139
|
* is retryable for the caller, who answers from the ledger by the same key;
|
|
@@ -132,6 +148,23 @@ class NotificationSettleError extends NotificationError {
|
|
|
132
148
|
});
|
|
133
149
|
}
|
|
134
150
|
}
|
|
151
|
+
/** The cache key of one resolver answer: topic, provider, target, locale, notice variant, and template data. */
|
|
152
|
+
function copyCacheKey(topic, provider, target, locale, origin, templateData) {
|
|
153
|
+
return JSON.stringify([
|
|
154
|
+
topic,
|
|
155
|
+
provider,
|
|
156
|
+
target ?? null,
|
|
157
|
+
locale ?? null,
|
|
158
|
+
origin.kind === 'notice' ? origin.variant : null,
|
|
159
|
+
templateData === undefined
|
|
160
|
+
? null
|
|
161
|
+
: Object.entries(templateData).sort(([left], [right]) => (left < right
|
|
162
|
+
? -1
|
|
163
|
+
: left > right
|
|
164
|
+
? 1
|
|
165
|
+
: 0)),
|
|
166
|
+
]);
|
|
167
|
+
}
|
|
135
168
|
export function createNotificationService(input) {
|
|
136
169
|
const { store, registry, catalog } = input;
|
|
137
170
|
const deployment = input.deployment === undefined
|
|
@@ -147,6 +180,13 @@ export function createNotificationService(input) {
|
|
|
147
180
|
/** The claim lease after `attempts` ambiguous attempts: doubling, up to an hour (or the lease, if longer). */
|
|
148
181
|
const ambiguousLeaseMs = (attempts) => Math.min(leaseMs * 2 ** Math.min(attempts, 16), Math.max(leaseMs, maxAmbiguousLeaseMs));
|
|
149
182
|
const random = input.random ?? Math.random;
|
|
183
|
+
const resolveCopy = input.resolveCopy ?? resolveCatalogNotificationCopy;
|
|
184
|
+
/**
|
|
185
|
+
* A project's resolver answers text the backend cannot check ahead of the
|
|
186
|
+
* send: its placeholders are checked against the send's data when it
|
|
187
|
+
* answers, and a reminder's set time checks no catalog copy.
|
|
188
|
+
*/
|
|
189
|
+
const customCopy = input.resolveCopy !== undefined;
|
|
150
190
|
const noticeTopic = catalog.notices?.topic;
|
|
151
191
|
if (catalog.notices !== undefined) {
|
|
152
192
|
const issues = notificationNoticeCopyIssues(catalog.notices.copy, catalog.defaultLocale);
|
|
@@ -198,6 +238,131 @@ export function createNotificationService(input) {
|
|
|
198
238
|
}
|
|
199
239
|
return undefined;
|
|
200
240
|
};
|
|
241
|
+
/**
|
|
242
|
+
* The copy the resolver answers for a recipient's send. It is asked only
|
|
243
|
+
* once the gate, deciding with `copyStandIn`, allowed the recipient with
|
|
244
|
+
* content from copy (`notificationContentSource` is `catalog`), so the
|
|
245
|
+
* stand-in never reaches a provider.
|
|
246
|
+
*
|
|
247
|
+
* `cache` lives for one invocation (one `deliver`, one queued job, one
|
|
248
|
+
* `dispatchDue`, or one scheduler run) and keeps one answer per topic,
|
|
249
|
+
* provider, target, locale, notice variant, and template data, so the
|
|
250
|
+
* recipients and rows that share them share one resolver call, and a
|
|
251
|
+
* send's first check and its provider call in the same invocation do too.
|
|
252
|
+
* The template data is part of the key because the resolver is given it.
|
|
253
|
+
*
|
|
254
|
+
* A `NotificationError` the resolver throws keeps its own code and passes
|
|
255
|
+
* through (the Worker's cron budget does); any other throw is a copy
|
|
256
|
+
* outage (`NotificationCopyUnavailableError`). An answer that cannot be
|
|
257
|
+
* sent is reported and is no copy.
|
|
258
|
+
*/
|
|
259
|
+
const copyFor = async (recipient, request, origin, cache) => {
|
|
260
|
+
const topic = topicFor(recipient.provider, recipient.target, request.topic, origin);
|
|
261
|
+
if (topic === undefined) {
|
|
262
|
+
return {};
|
|
263
|
+
}
|
|
264
|
+
const provider = recipient.provider.entry.provider;
|
|
265
|
+
const locale = request.locale ?? recipient.locale;
|
|
266
|
+
const key = copyCacheKey(topic.id, provider, recipient.target, locale, origin, request.templateData);
|
|
267
|
+
let resolved = cache.get(key);
|
|
268
|
+
if (resolved === undefined) {
|
|
269
|
+
resolved = (async () => {
|
|
270
|
+
let resolution;
|
|
271
|
+
try {
|
|
272
|
+
resolution = await resolveNotificationCopy(resolveCopy, {
|
|
273
|
+
topic,
|
|
274
|
+
provider,
|
|
275
|
+
...(locale === undefined ? {} : { locale }),
|
|
276
|
+
defaultLocale: catalog.defaultLocale,
|
|
277
|
+
...(request.templateData === undefined ? {} : { templateData: request.templateData }),
|
|
278
|
+
...(origin.kind === 'notice' ? { notice: { variant: origin.variant } } : {}),
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (error instanceof NotificationError) {
|
|
283
|
+
throw error;
|
|
284
|
+
}
|
|
285
|
+
throw new NotificationCopyUnavailableError(topic.id, { cause: error });
|
|
286
|
+
}
|
|
287
|
+
return checkedCopy(resolution, topic.id, request.idempotencyKey, provider, locale, request.templateData);
|
|
288
|
+
})();
|
|
289
|
+
cache.set(key, resolved);
|
|
290
|
+
}
|
|
291
|
+
return resolved;
|
|
292
|
+
};
|
|
293
|
+
/** A resolver's answer as the gate takes it: no copy (reported) when it is missing or cannot be filled. */
|
|
294
|
+
const checkedCopy = (resolution, topicId, sendKey, provider, locale, templateData) => {
|
|
295
|
+
const where = { sendKey, provider };
|
|
296
|
+
if (resolution.copy === undefined) {
|
|
297
|
+
// Reported, so a missing translation is not only a quiet copy-missing.
|
|
298
|
+
report(new TypeError(`Topic ${topicId}: ${resolution.issue ?? `no copy for ${locale ?? catalog.defaultLocale}`}.`), 'copy.resolve', where);
|
|
299
|
+
return {};
|
|
300
|
+
}
|
|
301
|
+
if (customCopy) {
|
|
302
|
+
try {
|
|
303
|
+
fillNotificationTemplate(resolution.copy.title, templateData);
|
|
304
|
+
fillNotificationTemplate(resolution.copy.body, templateData);
|
|
305
|
+
}
|
|
306
|
+
catch (error) {
|
|
307
|
+
// The project's text needs a value this send does not carry.
|
|
308
|
+
report(new TypeError(`Topic ${topicId}: ${describe(error)}`), 'copy.resolve', where);
|
|
309
|
+
return {};
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return { copy: resolution.copy };
|
|
313
|
+
};
|
|
314
|
+
/**
|
|
315
|
+
* A bulk send's copy, up front (BK1): one `resolveNotificationCopyByLocale`
|
|
316
|
+
* call for the locales its recipients name (the send's `locale`, or each
|
|
317
|
+
* recipient's own), stored in the invocation's `cache` under the keys
|
|
318
|
+
* `copyFor` reads, so planning, the provider call, and the flush-time
|
|
319
|
+
* gate check share each locale group's one answer. A recipient whose
|
|
320
|
+
* locale comes from its registration joins its group through the same
|
|
321
|
+
* cache. Only for a platform-user provider whose content comes from copy
|
|
322
|
+
* (a host template or a mapping's own text asks no resolver). It runs
|
|
323
|
+
* when the first recipient needs a send (`ensureCopies` in `deliverBulk`),
|
|
324
|
+
* so a retry whose recipients were all sent never asks the resolver. A
|
|
325
|
+
* resolver that throws here is reported and caches nothing: each locale
|
|
326
|
+
* group is then asked when a recipient the gate allows needs it, and a
|
|
327
|
+
* failure there fails that group's recipients, retryable.
|
|
328
|
+
*/
|
|
329
|
+
const primeBulkCopies = async (provider, request, locales, cache) => {
|
|
330
|
+
if (provider.entry.addressing !== 'platform-user') {
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
const origin = { kind: 'send' };
|
|
334
|
+
const topic = topicFor(provider, undefined, request.topic, origin);
|
|
335
|
+
if (topic === undefined || notificationContentSource(topic, provider.entry.content) !== 'catalog') {
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
const id = provider.entry.provider;
|
|
339
|
+
const keyOf = (locale) => copyCacheKey(topic.id, id, undefined, locale, origin, request.templateData);
|
|
340
|
+
const wanted = [...new Set(locales)].filter((locale) => !cache.has(keyOf(locale)));
|
|
341
|
+
if (wanted.length === 0) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
let resolved;
|
|
345
|
+
try {
|
|
346
|
+
resolved = await resolveNotificationCopyByLocale(resolveCopy, {
|
|
347
|
+
topic,
|
|
348
|
+
provider: id,
|
|
349
|
+
defaultLocale: catalog.defaultLocale,
|
|
350
|
+
...(request.templateData === undefined ? {} : { templateData: request.templateData }),
|
|
351
|
+
locales: wanted,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
// The up-front answers are an optimization: a failure caches nothing,
|
|
356
|
+
// so each locale group is asked again only when a recipient the gate
|
|
357
|
+
// allows needs it (`copyFor`), and a locale of refused recipients never
|
|
358
|
+
// fails the others.
|
|
359
|
+
report(error, 'copy.resolve.bulk', { sendKey: request.idempotencyKey, provider: id });
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
for (const locale of wanted) {
|
|
363
|
+
cache.set(keyOf(locale), Promise.resolve(checkedCopy(resolved.get(locale) ?? {}, topic.id, request.idempotencyKey, id, locale, request.templateData)));
|
|
364
|
+
}
|
|
365
|
+
};
|
|
201
366
|
const refuseNoticeTopic = (topic) => {
|
|
202
367
|
if (noticeTopic !== undefined && topic === noticeTopic) {
|
|
203
368
|
throw invalidInput(`Topic ${topic} is reserved for consent notices, which only the scheduler sends.`);
|
|
@@ -415,7 +580,7 @@ export function createNotificationService(input) {
|
|
|
415
580
|
? early
|
|
416
581
|
: undefined;
|
|
417
582
|
};
|
|
418
|
-
const decide = (request, origin, recipient, consent, history, serverPermission, at) => {
|
|
583
|
+
const decide = (request, origin, recipient, consent, history, serverPermission, at, resolvedCopy) => {
|
|
419
584
|
const topic = topicFor(recipient.provider, recipient.target, request.topic, origin);
|
|
420
585
|
const locale = request.locale ?? recipient.locale;
|
|
421
586
|
try {
|
|
@@ -437,6 +602,7 @@ export function createNotificationService(input) {
|
|
|
437
602
|
...(locale === undefined ? {} : { locale }),
|
|
438
603
|
defaultLocale: catalog.defaultLocale,
|
|
439
604
|
...(request.templateData === undefined ? {} : { templateData: request.templateData }),
|
|
605
|
+
...(resolvedCopy === undefined ? {} : { resolvedCopy }),
|
|
440
606
|
}),
|
|
441
607
|
};
|
|
442
608
|
}
|
|
@@ -488,7 +654,8 @@ export function createNotificationService(input) {
|
|
|
488
654
|
state: { since: formatNotificationInstant(since), delaySeconds, attempts: attempts + 1 },
|
|
489
655
|
};
|
|
490
656
|
};
|
|
491
|
-
|
|
657
|
+
/** What the provider is asked to send for one allowed recipient. */
|
|
658
|
+
const deliveryRequestOf = (recipient, request, topic, decision, at) => {
|
|
492
659
|
const mappedLink = topic.mapping?.['link'];
|
|
493
660
|
const link = request.link ?? (typeof mappedLink === 'string' ? mappedLink : undefined);
|
|
494
661
|
const target = recipient.endpoint !== undefined
|
|
@@ -499,36 +666,45 @@ export function createNotificationService(input) {
|
|
|
499
666
|
}
|
|
500
667
|
: { kind: 'platform-user', id: recipient.platformId ?? '' };
|
|
501
668
|
const locale = request.locale ?? recipient.locale;
|
|
669
|
+
return {
|
|
670
|
+
idempotencyKey: recipient.rowKey,
|
|
671
|
+
sendKey: request.idempotencyKey,
|
|
672
|
+
provider: recipient.provider.entry.provider,
|
|
673
|
+
player: request.player,
|
|
674
|
+
topic,
|
|
675
|
+
recipient: target,
|
|
676
|
+
content: decision.content,
|
|
677
|
+
...(decision.expiresAt === undefined ? {} : { expiresAt: decision.expiresAt }),
|
|
678
|
+
...(request.templateData === undefined ? {} : { templateData: request.templateData }),
|
|
679
|
+
...(request.data === undefined ? {} : { data: request.data }),
|
|
680
|
+
...(link === undefined ? {} : { link }),
|
|
681
|
+
...(locale === undefined ? {} : { locale }),
|
|
682
|
+
at: formatNotificationInstant(at),
|
|
683
|
+
};
|
|
684
|
+
};
|
|
685
|
+
/** One recipient's provider call (`send`), as one of the five answers. */
|
|
686
|
+
const sendOne = async (provider, delivery) => {
|
|
502
687
|
try {
|
|
503
|
-
const answer = await
|
|
504
|
-
idempotencyKey: recipient.rowKey,
|
|
505
|
-
sendKey: request.idempotencyKey,
|
|
506
|
-
provider: recipient.provider.entry.provider,
|
|
507
|
-
player: request.player,
|
|
508
|
-
topic,
|
|
509
|
-
recipient: target,
|
|
510
|
-
content: decision.content,
|
|
511
|
-
...(decision.expiresAt === undefined ? {} : { expiresAt: decision.expiresAt }),
|
|
512
|
-
...(request.templateData === undefined ? {} : { templateData: request.templateData }),
|
|
513
|
-
...(request.data === undefined ? {} : { data: request.data }),
|
|
514
|
-
...(link === undefined ? {} : { link }),
|
|
515
|
-
...(locale === undefined ? {} : { locale }),
|
|
516
|
-
at: formatNotificationInstant(at),
|
|
517
|
-
});
|
|
688
|
+
const answer = await provider.delivery.send(delivery);
|
|
518
689
|
const known = deliveryAnswerOf(answer);
|
|
519
690
|
if (known === undefined) {
|
|
520
691
|
// A provider that broke the contract may still have sent it.
|
|
521
|
-
report(new TypeError('The provider answered no known delivery answer.'), 'provider.send',
|
|
692
|
+
report(new TypeError('The provider answered no known delivery answer.'), 'provider.send', delivery.sendKey);
|
|
522
693
|
return { status: 'ambiguous' };
|
|
523
694
|
}
|
|
524
695
|
return known;
|
|
525
696
|
}
|
|
526
697
|
catch (error) {
|
|
527
698
|
// The provider may have sent it: one rule for every failure it cannot prove.
|
|
528
|
-
report(error, 'provider.send',
|
|
699
|
+
report(error, 'provider.send', delivery.sendKey, true);
|
|
529
700
|
return { status: 'ambiguous' };
|
|
530
701
|
}
|
|
531
702
|
};
|
|
703
|
+
const callProvider = (recipient, request, topic, decision, at,
|
|
704
|
+
// When the caller's claim on the row ends (epoch ms); a bulk batch checks it before its call.
|
|
705
|
+
_claimedUntil,
|
|
706
|
+
// The gate again at a later instant; a bulk batch runs it just before its call.
|
|
707
|
+
_regate) => sendOne(recipient.provider, deliveryRequestOf(recipient, request, topic, decision, at));
|
|
532
708
|
/**
|
|
533
709
|
* The rows a new send stores, after a first gate check per recipient:
|
|
534
710
|
* allowed recipients become queued rows, deferred ones deferred rows, and
|
|
@@ -537,7 +713,7 @@ export function createNotificationService(input) {
|
|
|
537
713
|
* at `queuedUntil` for a scheduler item, whose rows are due at once and
|
|
538
714
|
* leave through the paced due path.
|
|
539
715
|
*/
|
|
540
|
-
const plan = async (request, origin, at, queuedUntil) => {
|
|
716
|
+
const plan = async (request, origin, at, queuedUntil, copies = new Map()) => {
|
|
541
717
|
const recipients = await recipientsOf(request, at, origin);
|
|
542
718
|
if (recipients.length === 0) {
|
|
543
719
|
return { rows: [], refusals: [noEndpoint()] };
|
|
@@ -557,20 +733,32 @@ export function createNotificationService(input) {
|
|
|
557
733
|
perProvider.set(id, inputs);
|
|
558
734
|
}
|
|
559
735
|
const held = inputs;
|
|
560
|
-
const evaluate = (permission) => decide(request, origin, recipient, consent, held.history, permission, at);
|
|
561
736
|
// One read per provider and send, made only once a recipient needs it.
|
|
562
737
|
// A host that cannot answer is skipped; the send fails only when no
|
|
563
|
-
// provider could be checked and none passed.
|
|
738
|
+
// provider could be checked and none passed. The copy resolver is
|
|
739
|
+
// asked only for a recipient the gate would allow, once per target and
|
|
740
|
+
// locale. Its failure fails the whole check rather than skipping the
|
|
741
|
+
// recipients of one locale, which would store a send that never
|
|
742
|
+
// reaches them.
|
|
564
743
|
let decision;
|
|
744
|
+
let resolvingCopy = false;
|
|
565
745
|
try {
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
746
|
+
const evaluate = (permission, resolvedCopy = copyStandIn) => decide(request, origin, recipient, consent, held.history, permission, at, resolvedCopy);
|
|
747
|
+
const early = earlyRefusal(recipient.provider, (standIn) => evaluate(standIn));
|
|
748
|
+
const permission = early === undefined
|
|
749
|
+
? await (held.permission ??= readServerPermission(recipient.provider, request))
|
|
750
|
+
: undefined;
|
|
751
|
+
decision = early?.decision ?? evaluate(permission).decision;
|
|
752
|
+
if (decision.status === 'allowed' && decision.content.source === 'catalog') {
|
|
753
|
+
resolvingCopy = true;
|
|
754
|
+
decision = evaluate(permission, await copyFor(recipient, request, origin, copies)).decision;
|
|
755
|
+
}
|
|
570
756
|
checked += 1;
|
|
571
757
|
}
|
|
572
758
|
catch (error) {
|
|
573
|
-
if (
|
|
759
|
+
if (!resolvingCopy
|
|
760
|
+
&& error instanceof NotificationError
|
|
761
|
+
&& error.code === 'NOTIFICATIONS_TEMPORARILY_UNAVAILABLE') {
|
|
574
762
|
hostFailure ??= error;
|
|
575
763
|
continue;
|
|
576
764
|
}
|
|
@@ -686,7 +874,9 @@ export function createNotificationService(input) {
|
|
|
686
874
|
* A notice the game recorded since the claim, or a reminder cancelled or
|
|
687
875
|
* replaced since the claim, is dropped instead of sent.
|
|
688
876
|
*/
|
|
689
|
-
const runRow = async (sendKey, provider, rowKey, mode)
|
|
877
|
+
const runRow = async (sendKey, provider, rowKey, mode, copies = new Map(),
|
|
878
|
+
// A bulk send gathers its recipients' calls into one provider call.
|
|
879
|
+
call = callProvider) => {
|
|
690
880
|
let claimed;
|
|
691
881
|
for (let attempt = 0; attempt < maxLedgerAttempts && claimed === undefined; attempt += 1) {
|
|
692
882
|
const at = nowMs();
|
|
@@ -757,15 +947,35 @@ export function createNotificationService(input) {
|
|
|
757
947
|
let evaluated;
|
|
758
948
|
let recipient;
|
|
759
949
|
let at;
|
|
950
|
+
// The same gate at a later instant, with the inputs read now: a bulk batch checks it again before its call.
|
|
951
|
+
let regate;
|
|
760
952
|
try {
|
|
761
953
|
recipient = await recipientOfRow(registered, send.request, row, claimed.at);
|
|
762
954
|
const consent = await store.listConsent(send.request.player);
|
|
763
|
-
const early = earlyRefusal(registered, (standIn) => decide(send.request, send.origin, recipient, consent, [], standIn, nowMs()));
|
|
955
|
+
const early = earlyRefusal(registered, (standIn) => decide(send.request, send.origin, recipient, consent, [], standIn, nowMs(), copyStandIn));
|
|
764
956
|
const serverPermission = early === undefined
|
|
765
957
|
? await readServerPermission(registered, send.request)
|
|
766
958
|
: undefined;
|
|
767
959
|
at = nowMs();
|
|
768
|
-
|
|
960
|
+
const history = early === undefined
|
|
961
|
+
? historyOn(await loadHistory(send.request.player, sendKey, at, { key: sendKey, claimedAt: claimed.at }), row.provider)
|
|
962
|
+
: [];
|
|
963
|
+
const evaluate = (resolvedCopy) => decide(send.request, send.origin, recipient, consent, history, serverPermission, at, resolvedCopy);
|
|
964
|
+
evaluated = early ?? evaluate(copyStandIn);
|
|
965
|
+
let gatedCopy = copyStandIn;
|
|
966
|
+
// The copy resolver is asked only once the gate would allow the call.
|
|
967
|
+
if (early === undefined
|
|
968
|
+
&& evaluated.decision.status === 'allowed'
|
|
969
|
+
&& evaluated.decision.content.source === 'catalog') {
|
|
970
|
+
gatedCopy = await copyFor(recipient, send.request, send.origin, copies);
|
|
971
|
+
// The resolver took time: the gate decides again at the time of the
|
|
972
|
+
// call, so a window that opened meanwhile (the night window) holds it.
|
|
973
|
+
at = nowMs();
|
|
974
|
+
evaluated = evaluate(gatedCopy);
|
|
975
|
+
}
|
|
976
|
+
const gated = recipient;
|
|
977
|
+
const copy = gatedCopy;
|
|
978
|
+
regate = (when) => decide(send.request, send.origin, gated, consent, history, serverPermission, when, copy).decision;
|
|
769
979
|
}
|
|
770
980
|
catch (error) {
|
|
771
981
|
if (error instanceof NotificationSendInputError) {
|
|
@@ -812,7 +1022,7 @@ export function createNotificationService(input) {
|
|
|
812
1022
|
await complete(sendKey, row, token, { kind: 'remove' }, nowMs());
|
|
813
1023
|
return {};
|
|
814
1024
|
}
|
|
815
|
-
const answer = await
|
|
1025
|
+
const answer = await call(recipient, send.request, topic, decision, at, row.state.status === 'in-flight' ? Date.parse(row.state.leaseUntil) : undefined, regate);
|
|
816
1026
|
const answeredAt = nowMs();
|
|
817
1027
|
switch (answer.status) {
|
|
818
1028
|
case 'accepted':
|
|
@@ -839,7 +1049,10 @@ export function createNotificationService(input) {
|
|
|
839
1049
|
const leaseUntil = answeredAt + ambiguousLeaseMs(ambiguousAttempts);
|
|
840
1050
|
// The replay window started at the claim that reached the provider (or an earlier one).
|
|
841
1051
|
const firstAttemptedAt = row.firstAttemptedAt ?? formatNotificationInstant(at);
|
|
842
|
-
if (
|
|
1052
|
+
if (answer.unattributed === true) {
|
|
1053
|
+
report(new Error(`A bulk call reached some of its recipients without naming them${answer.providerCode === undefined ? '' : ` (${answer.providerCode})`}: the delivery stays in flight and is not replayed, so no recipient gets it twice.`), 'dispatch.unattributed', { sendKey, provider: row.provider, rowKey: row.key });
|
|
1054
|
+
}
|
|
1055
|
+
else if (isNotificationRowAbandoned({ ...row, firstAttemptedAt }, leaseUntil)) {
|
|
843
1056
|
report(new Error(answer.providerCode === undefined
|
|
844
1057
|
? 'Ambiguous answers for a day: the delivery stays in flight and is not replayed again.'
|
|
845
1058
|
: `Ambiguous answers for a day (${answer.providerCode}): the delivery stays in flight and is not replayed again.`), 'dispatch.abandoned', { sendKey, provider: row.provider, rowKey: row.key });
|
|
@@ -851,6 +1064,7 @@ export function createNotificationService(input) {
|
|
|
851
1064
|
...backoff,
|
|
852
1065
|
firstAttemptedAt,
|
|
853
1066
|
...(ambiguousAttempts === 0 ? {} : { ambiguousAttempts }),
|
|
1067
|
+
...(answer.unattributed === true ? { unattributed: true } : {}),
|
|
854
1068
|
state: {
|
|
855
1069
|
status: 'in-flight',
|
|
856
1070
|
leaseUntil: formatNotificationInstant(leaseUntil),
|
|
@@ -885,6 +1099,12 @@ export function createNotificationService(input) {
|
|
|
885
1099
|
}
|
|
886
1100
|
await complete(sendKey, row, token, { kind: 'remove' }, answeredAt);
|
|
887
1101
|
return { refusal: { status: 'unavailable', reason: 'provider-rejected' } };
|
|
1102
|
+
case 'not-sent':
|
|
1103
|
+
// No call was made for it (a bulk batch left it out): the row goes
|
|
1104
|
+
// back as it was before this claim, due again shortly. It is not a
|
|
1105
|
+
// rate limit, so it neither backs off nor nears the 24-hour give-up.
|
|
1106
|
+
await complete(sendKey, row, token, { kind: 'set', row: backedOff(before, answeredAt) }, answeredAt);
|
|
1107
|
+
return {};
|
|
888
1108
|
case 'rate-limited': {
|
|
889
1109
|
// Nothing was sent: the claim is released and the row waits.
|
|
890
1110
|
const next = nextRateLimit(row.rateLimit, answer.retryAfterSeconds, answeredAt);
|
|
@@ -957,13 +1177,13 @@ export function createNotificationService(input) {
|
|
|
957
1177
|
* got anywhere, so a caller never mistakes a send that did nothing for an
|
|
958
1178
|
* ambiguous one.
|
|
959
1179
|
*/
|
|
960
|
-
const runInline = async (sendKey, rows, mode) => {
|
|
1180
|
+
const runInline = async (sendKey, rows, mode, copies = new Map()) => {
|
|
961
1181
|
const refusals = [];
|
|
962
1182
|
let callerError;
|
|
963
1183
|
const errors = [];
|
|
964
1184
|
for (const row of rows) {
|
|
965
1185
|
try {
|
|
966
|
-
const { refusal } = await runRow(sendKey, row.provider, row.key, mode);
|
|
1186
|
+
const { refusal } = await runRow(sendKey, row.provider, row.key, mode, copies);
|
|
967
1187
|
if (refusal !== undefined) {
|
|
968
1188
|
refusals.push(refusal);
|
|
969
1189
|
}
|
|
@@ -996,9 +1216,11 @@ export function createNotificationService(input) {
|
|
|
996
1216
|
};
|
|
997
1217
|
/** Evaluates a send this call created, holding its first-evaluation claim. */
|
|
998
1218
|
const sendNew = async (send, version, at) => {
|
|
1219
|
+
// One copy cache for the first check and the inline provider calls.
|
|
1220
|
+
const copies = new Map();
|
|
999
1221
|
let planned;
|
|
1000
1222
|
try {
|
|
1001
|
-
planned = await plan(send.request, send.origin, at);
|
|
1223
|
+
planned = await plan(send.request, send.origin, at, undefined, copies);
|
|
1002
1224
|
}
|
|
1003
1225
|
catch (error) {
|
|
1004
1226
|
// Nothing was sent: the key is released so a retry evaluates again.
|
|
@@ -1039,7 +1261,7 @@ export function createNotificationService(input) {
|
|
|
1039
1261
|
}
|
|
1040
1262
|
return notificationSendOutcome(stored) ?? { status: 'in-flight' };
|
|
1041
1263
|
}
|
|
1042
|
-
const ran = await runInline(send.key, queued, 'job');
|
|
1264
|
+
const ran = await runInline(send.key, queued, 'job', copies);
|
|
1043
1265
|
return settle(send.key, [...refusals, ...ran.refusals]);
|
|
1044
1266
|
};
|
|
1045
1267
|
// Due rows (the scheduler's dispatch).
|
|
@@ -1107,6 +1329,13 @@ export function createNotificationService(input) {
|
|
|
1107
1329
|
budgets.set(provider, left - 1);
|
|
1108
1330
|
return left > 0;
|
|
1109
1331
|
},
|
|
1332
|
+
/** Gives back one `take` that made no delivery after all. */
|
|
1333
|
+
refund(provider) {
|
|
1334
|
+
const left = budgets.get(provider);
|
|
1335
|
+
if (left !== undefined) {
|
|
1336
|
+
budgets.set(provider, left + 1);
|
|
1337
|
+
}
|
|
1338
|
+
},
|
|
1110
1339
|
};
|
|
1111
1340
|
};
|
|
1112
1341
|
/**
|
|
@@ -1116,13 +1345,18 @@ export function createNotificationService(input) {
|
|
|
1116
1345
|
* Answers how many sends were listed and how many of them the pace held
|
|
1117
1346
|
* back entirely.
|
|
1118
1347
|
*/
|
|
1119
|
-
const dispatchDueRows = async (options = {}, pace = paceBudgets()) => {
|
|
1348
|
+
const dispatchDueRows = async (options = {}, pace = paceBudgets(), copies = new Map()) => {
|
|
1120
1349
|
const limit = batchLimit(options.limit);
|
|
1121
1350
|
const keys = await store.listDueSends({ at: formatNotificationInstant(nowMs()), limit });
|
|
1122
1351
|
const listed = keys.length;
|
|
1123
|
-
|
|
1352
|
+
// Each listed send's result, in listing order; a failed one has none.
|
|
1353
|
+
const slots = new Map();
|
|
1124
1354
|
let failed = 0;
|
|
1125
1355
|
let paced = 0;
|
|
1356
|
+
// Sends whose due rows all go to one provider with a bulk path: replayed
|
|
1357
|
+
// together through its `sendBatch` (BK1), each row still its own claim,
|
|
1358
|
+
// gate check, and answer.
|
|
1359
|
+
const gathered = new Map();
|
|
1126
1360
|
for (const sendKey of keys) {
|
|
1127
1361
|
try {
|
|
1128
1362
|
const held = await store.getSend(sendKey);
|
|
@@ -1160,20 +1394,52 @@ export function createNotificationService(input) {
|
|
|
1160
1394
|
failed += 1;
|
|
1161
1395
|
report(error, 'queue.enqueue', sendKey);
|
|
1162
1396
|
}
|
|
1163
|
-
|
|
1397
|
+
slots.set(sendKey, { sendKey, outcome: await settle(sendKey, []) });
|
|
1164
1398
|
continue;
|
|
1165
1399
|
}
|
|
1166
|
-
const
|
|
1400
|
+
const only = due.every((row) => row.provider === due[0]?.provider)
|
|
1401
|
+
? registry.provider(due[0]?.provider ?? '')
|
|
1402
|
+
: undefined;
|
|
1403
|
+
if (only?.delivery.bulk !== undefined && only.delivery.sendBatch !== undefined) {
|
|
1404
|
+
const group = gathered.get(only.entry.provider)
|
|
1405
|
+
?? { provider: only, sends: [] };
|
|
1406
|
+
group.sends.push({ key: sendKey, rows: due });
|
|
1407
|
+
gathered.set(only.entry.provider, group);
|
|
1408
|
+
slots.set(sendKey, undefined);
|
|
1409
|
+
continue;
|
|
1410
|
+
}
|
|
1411
|
+
const ran = await runInline(sendKey, due, 'due', copies);
|
|
1167
1412
|
if (ran.failed > 0) {
|
|
1168
1413
|
failed += 1;
|
|
1169
1414
|
}
|
|
1170
|
-
|
|
1415
|
+
slots.set(sendKey, { sendKey, outcome: await settle(sendKey, ran.refusals) });
|
|
1171
1416
|
}
|
|
1172
1417
|
catch (error) {
|
|
1173
1418
|
failed += 1;
|
|
1174
1419
|
report(error, 'dispatchDue', sendKey);
|
|
1175
1420
|
}
|
|
1176
1421
|
}
|
|
1422
|
+
for (const { provider, sends } of gathered.values()) {
|
|
1423
|
+
let answers;
|
|
1424
|
+
try {
|
|
1425
|
+
answers = await runBulkBatch(provider, sends.map((send) => ({ key: send.key, rows: send.rows, mode: 'due', refusals: [], wasDelivered: false })), { batch: 0, single: 0 }, copies);
|
|
1426
|
+
}
|
|
1427
|
+
catch (error) {
|
|
1428
|
+
failed += sends.length;
|
|
1429
|
+
report(error, 'dispatchDue', sends[0]?.key ?? '');
|
|
1430
|
+
continue;
|
|
1431
|
+
}
|
|
1432
|
+
sends.forEach((send, index) => {
|
|
1433
|
+
const answer = answers[index];
|
|
1434
|
+
if (answer !== undefined && 'outcome' in answer) {
|
|
1435
|
+
slots.set(send.key, { sendKey: send.key, outcome: answer.outcome });
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
failed += 1;
|
|
1439
|
+
report(answer !== undefined && 'error' in answer ? answer.error : new Error('No answer.'), 'dispatchDue', send.key);
|
|
1440
|
+
});
|
|
1441
|
+
}
|
|
1442
|
+
const results = [...slots.values()].filter((result) => result !== undefined);
|
|
1177
1443
|
return { results, failed, listed, paced };
|
|
1178
1444
|
};
|
|
1179
1445
|
// Scheduler items.
|
|
@@ -1231,11 +1497,11 @@ export function createNotificationService(input) {
|
|
|
1231
1497
|
* releases the key and throws `NOTIFICATIONS_INVALID_INPUT`; any other
|
|
1232
1498
|
* failure releases it and rethrows.
|
|
1233
1499
|
*/
|
|
1234
|
-
const storeItemRows = async (created, at, dueAt) => {
|
|
1500
|
+
const storeItemRows = async (created, at, dueAt, copies = new Map()) => {
|
|
1235
1501
|
const { send, version } = created;
|
|
1236
1502
|
let planned;
|
|
1237
1503
|
try {
|
|
1238
|
-
planned = await plan(send.request, send.origin, at, Math.min(dueAt, at));
|
|
1504
|
+
planned = await plan(send.request, send.origin, at, Math.min(dueAt, at), copies);
|
|
1239
1505
|
}
|
|
1240
1506
|
catch (error) {
|
|
1241
1507
|
await releaseQuietly(send.key, version);
|
|
@@ -1372,7 +1638,7 @@ export function createNotificationService(input) {
|
|
|
1372
1638
|
* Answers `busy` while another run is making the send, and `stale` when
|
|
1373
1639
|
* this run read a reminder a set has replaced since: neither worked it.
|
|
1374
1640
|
*/
|
|
1375
|
-
const fireReminder = async (held, reminder) => {
|
|
1641
|
+
const fireReminder = async (held, reminder, copies = new Map()) => {
|
|
1376
1642
|
const { player } = held.entry;
|
|
1377
1643
|
const at = nowMs();
|
|
1378
1644
|
let key;
|
|
@@ -1445,7 +1711,7 @@ export function createNotificationService(input) {
|
|
|
1445
1711
|
// The rows are due when the entry was: a past fireAt counts from its
|
|
1446
1712
|
// set time. A pending reminder always has a due time (runScheduler
|
|
1447
1713
|
// takes only entries with one); fireAt only satisfies the type.
|
|
1448
|
-
result = await storeItemRows(made.created, at, Date.parse(held.entry.dueAt ?? reminder.fireAt));
|
|
1714
|
+
result = await storeItemRows(made.created, at, Date.parse(held.entry.dueAt ?? reminder.fireAt), copies);
|
|
1449
1715
|
}
|
|
1450
1716
|
catch (error) {
|
|
1451
1717
|
if (error instanceof NotificationError && error.code === 'NOTIFICATIONS_INVALID_INPUT') {
|
|
@@ -1538,7 +1804,7 @@ export function createNotificationService(input) {
|
|
|
1538
1804
|
* backoff after a refused push (an hour, doubling to a day). With nothing
|
|
1539
1805
|
* owed it is removed.
|
|
1540
1806
|
*/
|
|
1541
|
-
const checkNotices = async (held, check) => {
|
|
1807
|
+
const checkNotices = async (held, check, copies = new Map()) => {
|
|
1542
1808
|
const { player } = held.entry;
|
|
1543
1809
|
if (!carriesNotices() || noticeTopic === undefined) {
|
|
1544
1810
|
await removeEntry(held);
|
|
@@ -1613,7 +1879,7 @@ export function createNotificationService(input) {
|
|
|
1613
1879
|
continue;
|
|
1614
1880
|
}
|
|
1615
1881
|
try {
|
|
1616
|
-
const result = await storeItemRows(made.created, at, seoulMidnight(sendFrom));
|
|
1882
|
+
const result = await storeItemRows(made.created, at, seoulMidnight(sendFrom), copies);
|
|
1617
1883
|
if (result === undefined) {
|
|
1618
1884
|
// Another run took the key over: its push is looked at again.
|
|
1619
1885
|
waiting = true;
|
|
@@ -1803,7 +2069,7 @@ export function createNotificationService(input) {
|
|
|
1803
2069
|
* checked, or forgotten, with each send it touched. A schedule that cannot
|
|
1804
2070
|
* be read is counted and reported, and the round's due rows still go out.
|
|
1805
2071
|
*/
|
|
1806
|
-
const runScheduleEntries = async (limit, touched) => {
|
|
2072
|
+
const runScheduleEntries = async (limit, touched, copies = new Map()) => {
|
|
1807
2073
|
let failed = 0;
|
|
1808
2074
|
let due = [];
|
|
1809
2075
|
try {
|
|
@@ -1824,7 +2090,7 @@ export function createNotificationService(input) {
|
|
|
1824
2090
|
}
|
|
1825
2091
|
const { item } = held.entry;
|
|
1826
2092
|
if (item.kind === 'notices') {
|
|
1827
|
-
for (const result of await checkNotices(held, item)) {
|
|
2093
|
+
for (const result of await checkNotices(held, item, copies)) {
|
|
1828
2094
|
touched.set(result.sendKey, result.outcome);
|
|
1829
2095
|
}
|
|
1830
2096
|
}
|
|
@@ -1832,7 +2098,7 @@ export function createNotificationService(input) {
|
|
|
1832
2098
|
await forgetFiredReminder(held, item);
|
|
1833
2099
|
}
|
|
1834
2100
|
else {
|
|
1835
|
-
const result = await fireReminder(held, item);
|
|
2101
|
+
const result = await fireReminder(held, item, copies);
|
|
1836
2102
|
if (result === 'busy') {
|
|
1837
2103
|
// Another run holds its send, or a live claim of other content
|
|
1838
2104
|
// still occupies the key: it waits a minute, so the next round
|
|
@@ -1886,6 +2152,624 @@ export function createNotificationService(input) {
|
|
|
1886
2152
|
}
|
|
1887
2153
|
return current;
|
|
1888
2154
|
};
|
|
2155
|
+
// Bulk sends (RFC open question 5, BK1).
|
|
2156
|
+
/**
|
|
2157
|
+
* One recipient of a bulk send, made ready the way `deliver` makes a
|
|
2158
|
+
* send: answered from its key's state, or created, planned through the
|
|
2159
|
+
* gate, and stored with the rows to run now.
|
|
2160
|
+
*/
|
|
2161
|
+
const prepareBulkRecipient = async (raw, provider, pace,
|
|
2162
|
+
// Recipients this call may still attempt; one answered from the ledger costs none.
|
|
2163
|
+
work,
|
|
2164
|
+
// The call's copy answers, shared by every recipient of a locale group.
|
|
2165
|
+
copies,
|
|
2166
|
+
// Resolves the call's locale groups once, when the first recipient needs a send.
|
|
2167
|
+
ensureCopies) => {
|
|
2168
|
+
const request = sendRequestOf(raw);
|
|
2169
|
+
const fingerprint = notificationSendFingerprint(request);
|
|
2170
|
+
const key = request.idempotencyKey;
|
|
2171
|
+
for (let attempt = 0; attempt < maxLedgerAttempts; attempt += 1) {
|
|
2172
|
+
const at = nowMs();
|
|
2173
|
+
const held = await lookup(key, at);
|
|
2174
|
+
if (held !== undefined) {
|
|
2175
|
+
if (held.send.fingerprint !== fingerprint) {
|
|
2176
|
+
throw new NotificationError('NOTIFICATIONS_IDEMPOTENCY_CONFLICT', {
|
|
2177
|
+
message: `Key ${key} is stored for another request.`,
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
const { send } = held;
|
|
2181
|
+
if (send.rows.length === 0) {
|
|
2182
|
+
if (send.claim !== undefined && Date.parse(send.claim.leaseUntil) > at) {
|
|
2183
|
+
return { done: { outcome: { status: 'in-flight' } } };
|
|
2184
|
+
}
|
|
2185
|
+
await store.updateSend({ key, version: held.version, send: undefined });
|
|
2186
|
+
continue;
|
|
2187
|
+
}
|
|
2188
|
+
const wasDelivered = notificationSendOutcome(send)?.status === 'delivered';
|
|
2189
|
+
// A platform row whose lease ended (a call that stopped mid-way) is
|
|
2190
|
+
// this call's to replay, as a retried `deliver` replays it.
|
|
2191
|
+
const expired = send.rows.filter((row) => eligible('retry', row, at));
|
|
2192
|
+
if (expired.length > 0) {
|
|
2193
|
+
if (work.left <= 0) {
|
|
2194
|
+
return { done: { notAttempted: 'count-budget' } };
|
|
2195
|
+
}
|
|
2196
|
+
// Each delivery row is an attempt: a player's devices each cost one.
|
|
2197
|
+
work.left -= expired.length;
|
|
2198
|
+
for (const row of expired) {
|
|
2199
|
+
if (!await pace.take(row.provider)) {
|
|
2200
|
+
return { done: { notAttempted: 'provider-rate' } };
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
return { key, rows: expired, mode: 'retry', refusals: [], wasDelivered };
|
|
2204
|
+
}
|
|
2205
|
+
const stored = notificationSendOutcome(send) ?? noEndpoint();
|
|
2206
|
+
return {
|
|
2207
|
+
done: {
|
|
2208
|
+
outcome: stored.status === 'delivered'
|
|
2209
|
+
? { status: 'delivered', alreadyProcessed: true }
|
|
2210
|
+
: stored,
|
|
2211
|
+
},
|
|
2212
|
+
};
|
|
2213
|
+
}
|
|
2214
|
+
// Spent out of the call's budget only once the gate allowed a row.
|
|
2215
|
+
if (work.left <= 0) {
|
|
2216
|
+
return { done: { notAttempted: 'count-budget' } };
|
|
2217
|
+
}
|
|
2218
|
+
// The copy is resolved only once a recipient needs a send, so a retry
|
|
2219
|
+
// whose recipients were all sent never asks the resolver; a resolver
|
|
2220
|
+
// that fails fails this recipient, retryable, before its key is made.
|
|
2221
|
+
await ensureCopies();
|
|
2222
|
+
const send = {
|
|
2223
|
+
key,
|
|
2224
|
+
fingerprint,
|
|
2225
|
+
request,
|
|
2226
|
+
origin: { kind: 'send' },
|
|
2227
|
+
scheduled: false,
|
|
2228
|
+
rows: [],
|
|
2229
|
+
claim: { token: newToken(), leaseUntil: formatNotificationInstant(at + leaseMs) },
|
|
2230
|
+
changedAt: formatNotificationInstant(at),
|
|
2231
|
+
};
|
|
2232
|
+
const version = await store.createSend(send);
|
|
2233
|
+
if (version === undefined) {
|
|
2234
|
+
// A concurrent send created the key first: answer from its state.
|
|
2235
|
+
continue;
|
|
2236
|
+
}
|
|
2237
|
+
let planned;
|
|
2238
|
+
try {
|
|
2239
|
+
planned = await plan(send.request, send.origin, at, undefined, copies);
|
|
2240
|
+
}
|
|
2241
|
+
catch (error) {
|
|
2242
|
+
// Nothing was sent: the key is released so a retry evaluates again.
|
|
2243
|
+
await releaseQuietly(key, version);
|
|
2244
|
+
if (error instanceof NotificationSendInputError) {
|
|
2245
|
+
throw invalidInput(error.message, error);
|
|
2246
|
+
}
|
|
2247
|
+
throw error;
|
|
2248
|
+
}
|
|
2249
|
+
if (planned.rows.length === 0) {
|
|
2250
|
+
await store.updateSend({ key, version, send: undefined });
|
|
2251
|
+
return { done: { outcome: bestNotificationOutcome(planned.refusals) ?? noEndpoint() } };
|
|
2252
|
+
}
|
|
2253
|
+
// The call's and the provider's app-wide budgets are spent once the gate
|
|
2254
|
+
// allowed a delivery, and before any row is stored: a recipient over
|
|
2255
|
+
// either leaves nothing behind, and the same call retried sends it once
|
|
2256
|
+
// the budget frees. A refused recipient spends neither.
|
|
2257
|
+
const allowed = planned.rows.filter((row) => row.state.status === 'in-flight');
|
|
2258
|
+
if (allowed.length > 0) {
|
|
2259
|
+
// Taken at once, so recipients planned side by side cannot all pass it.
|
|
2260
|
+
let over;
|
|
2261
|
+
if (work.left <= 0) {
|
|
2262
|
+
over = 'count-budget';
|
|
2263
|
+
}
|
|
2264
|
+
else {
|
|
2265
|
+
work.left -= 1;
|
|
2266
|
+
let paced;
|
|
2267
|
+
try {
|
|
2268
|
+
paced = await pace.take(provider);
|
|
2269
|
+
}
|
|
2270
|
+
catch (error) {
|
|
2271
|
+
// The rate could not be read (the store failed): nothing was
|
|
2272
|
+
// sent, so the attempt goes back and the key is released.
|
|
2273
|
+
work.left += 1;
|
|
2274
|
+
await releaseQuietly(key, version);
|
|
2275
|
+
throw error;
|
|
2276
|
+
}
|
|
2277
|
+
if (!paced) {
|
|
2278
|
+
work.left += 1;
|
|
2279
|
+
over = 'provider-rate';
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
if (over !== undefined) {
|
|
2283
|
+
await releaseQuietly(key, version);
|
|
2284
|
+
return { done: { notAttempted: over } };
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
const { claim: _claim, ...unclaimed } = send;
|
|
2288
|
+
const stored = {
|
|
2289
|
+
...unclaimed,
|
|
2290
|
+
rows: planned.rows,
|
|
2291
|
+
changedAt: formatNotificationInstant(at),
|
|
2292
|
+
};
|
|
2293
|
+
if (!await store.updateSend({ key, version, send: stored })) {
|
|
2294
|
+
// Another caller took the key over: this call delivers nothing for it,
|
|
2295
|
+
// so the attempt and the provider's rate it reserved go back.
|
|
2296
|
+
if (allowed.length > 0) {
|
|
2297
|
+
work.left += 1;
|
|
2298
|
+
pace.refund(provider);
|
|
2299
|
+
}
|
|
2300
|
+
return { done: { outcome: { status: 'in-flight' } } };
|
|
2301
|
+
}
|
|
2302
|
+
// The first allowed row was counted above; each further device is one
|
|
2303
|
+
// more attempt and one more of the provider's rate. A device over
|
|
2304
|
+
// either stays queued: the scheduler sends it at its own pace.
|
|
2305
|
+
const now = allowed.slice(0, 1);
|
|
2306
|
+
for (const row of allowed.slice(1)) {
|
|
2307
|
+
if (work.left <= 0) {
|
|
2308
|
+
break;
|
|
2309
|
+
}
|
|
2310
|
+
work.left -= 1;
|
|
2311
|
+
let paced = false;
|
|
2312
|
+
try {
|
|
2313
|
+
paced = await pace.take(row.provider);
|
|
2314
|
+
}
|
|
2315
|
+
catch (error) {
|
|
2316
|
+
// The device's row is stored: it stays queued for the scheduler.
|
|
2317
|
+
report(error, 'deliverBulk.pace', {
|
|
2318
|
+
sendKey: key,
|
|
2319
|
+
provider: row.provider,
|
|
2320
|
+
rowKey: row.key,
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
if (!paced) {
|
|
2324
|
+
work.left += 1;
|
|
2325
|
+
break;
|
|
2326
|
+
}
|
|
2327
|
+
now.push(row);
|
|
2328
|
+
}
|
|
2329
|
+
return {
|
|
2330
|
+
key,
|
|
2331
|
+
rows: now,
|
|
2332
|
+
mode: 'job',
|
|
2333
|
+
refusals: planned.refusals,
|
|
2334
|
+
wasDelivered: false,
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
throw busy(key);
|
|
2338
|
+
};
|
|
2339
|
+
/**
|
|
2340
|
+
* The flush of one batch: the requests in the provider's call groups
|
|
2341
|
+
* (`bulk.callKey`, one title and body a call on Devvit), one `sendBatch`
|
|
2342
|
+
* per group and `maxRecipients`, or one `send` per request below
|
|
2343
|
+
* `minRecipients` and inside the provider's hold window. The gate runs
|
|
2344
|
+
* again for each request as its call starts, and no call starts past the
|
|
2345
|
+
* claims' deadline: a request the gate now refuses, or one past the
|
|
2346
|
+
* deadline, answers `not-sent` (it is replayed shortly). Every answer is one of the five kinds; a
|
|
2347
|
+
* missing or broken one, or a call that threw, is `ambiguous`.
|
|
2348
|
+
*/
|
|
2349
|
+
const flushBulk = async (provider, requests, calls,
|
|
2350
|
+
// Milliseconds from now in which the last provider call may start, so every claim outlives the calls.
|
|
2351
|
+
startWithinMs,
|
|
2352
|
+
// Each request as the gate allows it at a later instant (undefined: no longer).
|
|
2353
|
+
regates = []) => {
|
|
2354
|
+
const capability = provider.delivery.bulk;
|
|
2355
|
+
const startedAt = nowMs();
|
|
2356
|
+
const sendBatch = provider.delivery.sendBatch;
|
|
2357
|
+
const answers = requests.map(() => ({
|
|
2358
|
+
status: 'ambiguous',
|
|
2359
|
+
}));
|
|
2360
|
+
const notSent = { status: 'not-sent' };
|
|
2361
|
+
/** The request as the gate allows it at `at`, or `undefined`. */
|
|
2362
|
+
const current = (index, at) => {
|
|
2363
|
+
const regate = regates[index];
|
|
2364
|
+
try {
|
|
2365
|
+
return regate === undefined ? requests[index] : regate(at);
|
|
2366
|
+
}
|
|
2367
|
+
catch {
|
|
2368
|
+
return undefined;
|
|
2369
|
+
}
|
|
2370
|
+
};
|
|
2371
|
+
const singles = async (indexes) => {
|
|
2372
|
+
const limiter = createNotificationLimiter(bulkConcurrency);
|
|
2373
|
+
await Promise.all(indexes.map(async (index) => {
|
|
2374
|
+
await limiter.acquire();
|
|
2375
|
+
try {
|
|
2376
|
+
const at = nowMs();
|
|
2377
|
+
const request = at - startedAt >= startWithinMs ? undefined : current(index, at);
|
|
2378
|
+
if (request === undefined) {
|
|
2379
|
+
answers[index] = notSent;
|
|
2380
|
+
return;
|
|
2381
|
+
}
|
|
2382
|
+
calls.single += 1;
|
|
2383
|
+
answers[index] = await sendOne(provider, request);
|
|
2384
|
+
}
|
|
2385
|
+
finally {
|
|
2386
|
+
limiter.release();
|
|
2387
|
+
}
|
|
2388
|
+
}));
|
|
2389
|
+
};
|
|
2390
|
+
if (capability === undefined
|
|
2391
|
+
|| sendBatch === undefined
|
|
2392
|
+
|| requests.length < (capability.minRecipients ?? 2)
|
|
2393
|
+
|| isInsideNotificationHoldWindow(capability.holdWindow, nowMs())) {
|
|
2394
|
+
// Fewer than the minimum after the gate, or the hold window opened while the batch gathered.
|
|
2395
|
+
await singles(requests.map((_, index) => index));
|
|
2396
|
+
return answers;
|
|
2397
|
+
}
|
|
2398
|
+
const groups = new Map();
|
|
2399
|
+
requests.forEach((request, index) => {
|
|
2400
|
+
let key = '';
|
|
2401
|
+
try {
|
|
2402
|
+
key = capability.callKey?.(request) ?? '';
|
|
2403
|
+
}
|
|
2404
|
+
catch {
|
|
2405
|
+
key = '';
|
|
2406
|
+
}
|
|
2407
|
+
const group = groups.get(key) ?? [];
|
|
2408
|
+
group.push(index);
|
|
2409
|
+
groups.set(key, group);
|
|
2410
|
+
});
|
|
2411
|
+
for (const group of groups.values()) {
|
|
2412
|
+
for (let start = 0; start < group.length; start += capability.maxRecipients) {
|
|
2413
|
+
const at = nowMs();
|
|
2414
|
+
const left = startWithinMs - (at - startedAt);
|
|
2415
|
+
const sending = [];
|
|
2416
|
+
for (const index of group.slice(start, start + capability.maxRecipients)) {
|
|
2417
|
+
const request = left > 0 ? current(index, at) : undefined;
|
|
2418
|
+
if (request === undefined) {
|
|
2419
|
+
answers[index] = notSent;
|
|
2420
|
+
}
|
|
2421
|
+
else {
|
|
2422
|
+
sending.push({ index, request });
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
if (sending.length === 0) {
|
|
2426
|
+
continue;
|
|
2427
|
+
}
|
|
2428
|
+
if (sending.length < (capability.minRecipients ?? 2)) {
|
|
2429
|
+
await singles(sending.map((entry) => entry.index));
|
|
2430
|
+
continue;
|
|
2431
|
+
}
|
|
2432
|
+
const slice = sending.map((entry) => entry.request);
|
|
2433
|
+
calls.batch += 1;
|
|
2434
|
+
let raw;
|
|
2435
|
+
try {
|
|
2436
|
+
raw = await sendBatch.call(provider.delivery, slice, { startWithinMs: left });
|
|
2437
|
+
}
|
|
2438
|
+
catch (error) {
|
|
2439
|
+
// The provider may have sent any of them: one rule for every failure it cannot prove.
|
|
2440
|
+
report(error, 'provider.sendBatch', { sendKey: slice[0]?.sendKey ?? '', provider: provider.entry.provider }, true);
|
|
2441
|
+
continue;
|
|
2442
|
+
}
|
|
2443
|
+
const list = Array.isArray(raw) ? raw : [];
|
|
2444
|
+
if (list.length !== slice.length) {
|
|
2445
|
+
report(new TypeError(`The provider answered ${list.length} answers for a batch of ${slice.length}.`), 'provider.sendBatch', { sendKey: slice[0]?.sendKey ?? '', provider: provider.entry.provider });
|
|
2446
|
+
continue;
|
|
2447
|
+
}
|
|
2448
|
+
sending.forEach(({ index, request }, position) => {
|
|
2449
|
+
const known = deliveryAnswerOf(list[position]);
|
|
2450
|
+
if (known === undefined) {
|
|
2451
|
+
report(new TypeError('The provider answered no known delivery answer.'), 'provider.sendBatch', {
|
|
2452
|
+
sendKey: request.sendKey,
|
|
2453
|
+
provider: provider.entry.provider,
|
|
2454
|
+
rowKey: request.idempotencyKey,
|
|
2455
|
+
});
|
|
2456
|
+
}
|
|
2457
|
+
answers[index] = known ?? { status: 'ambiguous' };
|
|
2458
|
+
});
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
return answers;
|
|
2462
|
+
};
|
|
2463
|
+
/**
|
|
2464
|
+
* Runs the prepared recipients of one batch: each row through `runRow`
|
|
2465
|
+
* (claim, gate, provider call, settle), with their provider calls
|
|
2466
|
+
* gathered into `flushBulk` for a provider with a bulk capability. A
|
|
2467
|
+
* send's rows run one after the other, in rounds, so two rows of one send
|
|
2468
|
+
* never contend for its key. Answers each recipient as `deliver` would.
|
|
2469
|
+
*/
|
|
2470
|
+
const runBulkBatch = async (provider, prepared, calls, copies) => {
|
|
2471
|
+
const limiter = createNotificationLimiter(bulkConcurrency);
|
|
2472
|
+
const capability = provider.delivery.bulk;
|
|
2473
|
+
const state = prepared.map((entry) => ({
|
|
2474
|
+
refusals: [...entry.refusals],
|
|
2475
|
+
errors: [],
|
|
2476
|
+
callerError: undefined,
|
|
2477
|
+
settleFailed: false,
|
|
2478
|
+
}));
|
|
2479
|
+
const rounds = Math.max(0, ...prepared.map((entry) => entry.rows.length));
|
|
2480
|
+
for (let round = 0; round < rounds; round += 1) {
|
|
2481
|
+
const members = prepared
|
|
2482
|
+
.map((entry, index) => ({ entry, index, row: entry.rows[round] }))
|
|
2483
|
+
.filter((member) => member.row !== undefined);
|
|
2484
|
+
// Only a provider with a bulk path gathers calls, for a round of at
|
|
2485
|
+
// least its minimum and outside its hold window. Any other round sends
|
|
2486
|
+
// each recipient as its gate allows it and settles it at once, so no
|
|
2487
|
+
// claim waits on a queue of single calls past its lease.
|
|
2488
|
+
const gathers = capability !== undefined
|
|
2489
|
+
&& provider.delivery.sendBatch !== undefined
|
|
2490
|
+
&& members.length >= (capability.minRecipients ?? 2)
|
|
2491
|
+
&& !isInsideNotificationHoldWindow(capability.holdWindow, nowMs());
|
|
2492
|
+
const barrier = createNotificationBatchBarrier(members.length, async (items) => {
|
|
2493
|
+
// A recipient whose claim has less than half a lease left (it may run
|
|
2494
|
+
// out before the call answers, and another holder may then own its
|
|
2495
|
+
// row) is not sent. Nothing was sent, so it waits a moment and is
|
|
2496
|
+
// replayed (its own claim settles only while it still holds the row).
|
|
2497
|
+
// The gate runs again for each recipient at the batch's own time
|
|
2498
|
+
// (quiet hours or the night window may have begun while it gathered),
|
|
2499
|
+
// with the inputs its first check read. A recipient it no longer
|
|
2500
|
+
// allows is not sent either: it is replayed, and its replay's gate
|
|
2501
|
+
// defers or refuses it.
|
|
2502
|
+
const now = nowMs();
|
|
2503
|
+
const checked = items.map((item) => {
|
|
2504
|
+
if (item.claimedUntil - now < leaseMs / 2) {
|
|
2505
|
+
return { item, reason: 'has less than half a claim lease left' };
|
|
2506
|
+
}
|
|
2507
|
+
let request;
|
|
2508
|
+
try {
|
|
2509
|
+
request = item.regate(now);
|
|
2510
|
+
}
|
|
2511
|
+
catch {
|
|
2512
|
+
request = undefined;
|
|
2513
|
+
}
|
|
2514
|
+
return request === undefined
|
|
2515
|
+
? { item, reason: 'is no longer allowed by the gate, or its gate inputs are too old' }
|
|
2516
|
+
: { item, request };
|
|
2517
|
+
});
|
|
2518
|
+
const fresh = checked.flatMap((entry) => ('request' in entry ? [entry] : []));
|
|
2519
|
+
// The last call starts while every fresh claim still has a quarter lease left.
|
|
2520
|
+
const startWithinMs = Math.min(...fresh.map((entry) => entry.item.claimedUntil)) - leaseMs / 4 - now;
|
|
2521
|
+
const answers = await flushBulk(provider, fresh.map((entry) => entry.request), calls, startWithinMs, fresh.map((entry) => entry.item.regate));
|
|
2522
|
+
let next = 0;
|
|
2523
|
+
return checked.map((entry) => {
|
|
2524
|
+
if ('request' in entry) {
|
|
2525
|
+
next += 1;
|
|
2526
|
+
return answers[next - 1] ?? { status: 'ambiguous' };
|
|
2527
|
+
}
|
|
2528
|
+
report(new Error(`A bulk recipient ${entry.reason} when its batch is sent; it is not sent now.`), 'dispatch.stale', {
|
|
2529
|
+
sendKey: entry.item.request.sendKey,
|
|
2530
|
+
provider: provider.entry.provider,
|
|
2531
|
+
rowKey: entry.item.request.idempotencyKey,
|
|
2532
|
+
});
|
|
2533
|
+
return { status: 'not-sent' };
|
|
2534
|
+
});
|
|
2535
|
+
});
|
|
2536
|
+
await Promise.all(members.map(async ({ entry, index, row }) => {
|
|
2537
|
+
const slot = state[index];
|
|
2538
|
+
if (slot === undefined) {
|
|
2539
|
+
return;
|
|
2540
|
+
}
|
|
2541
|
+
await limiter.acquire();
|
|
2542
|
+
let holding = true;
|
|
2543
|
+
let arrived = false;
|
|
2544
|
+
const release = () => {
|
|
2545
|
+
if (holding) {
|
|
2546
|
+
holding = false;
|
|
2547
|
+
limiter.release();
|
|
2548
|
+
}
|
|
2549
|
+
};
|
|
2550
|
+
try {
|
|
2551
|
+
const { refusal } = await runRow(entry.key, row.provider, row.key, entry.mode, copies, async (recipient, request, topic, decision, at, claimedUntil, regate) => {
|
|
2552
|
+
arrived = true;
|
|
2553
|
+
if (!gathers) {
|
|
2554
|
+
calls.single += 1;
|
|
2555
|
+
return callProvider(recipient, request, topic, decision, at);
|
|
2556
|
+
}
|
|
2557
|
+
// The slot goes back while the batch gathers, so the barrier never waits on a recipient that cannot run.
|
|
2558
|
+
release();
|
|
2559
|
+
const answer = await barrier.arrive({
|
|
2560
|
+
request: deliveryRequestOf(recipient, request, topic, decision, at),
|
|
2561
|
+
claimedUntil: claimedUntil ?? at,
|
|
2562
|
+
regate: (when) => {
|
|
2563
|
+
// The consent, history, and host permission were read at `at`: past a short age
|
|
2564
|
+
// they may be stale, so the recipient is replayed with fresh reads instead.
|
|
2565
|
+
if (when - at > bulkGateFreshMs) {
|
|
2566
|
+
return undefined;
|
|
2567
|
+
}
|
|
2568
|
+
const again = regate?.(when);
|
|
2569
|
+
return again?.status === 'allowed'
|
|
2570
|
+
? deliveryRequestOf(recipient, request, topic, again, when)
|
|
2571
|
+
: undefined;
|
|
2572
|
+
},
|
|
2573
|
+
});
|
|
2574
|
+
await limiter.acquire();
|
|
2575
|
+
holding = true;
|
|
2576
|
+
return answer;
|
|
2577
|
+
});
|
|
2578
|
+
if (refusal !== undefined) {
|
|
2579
|
+
slot.refusals.push(refusal);
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
catch (error) {
|
|
2583
|
+
if (error instanceof NotificationError && error.code === 'NOTIFICATIONS_INVALID_INPUT') {
|
|
2584
|
+
slot.callerError ??= error;
|
|
2585
|
+
}
|
|
2586
|
+
else {
|
|
2587
|
+
slot.errors.push({ error, rowKey: row.key, provider: row.provider });
|
|
2588
|
+
slot.settleFailed ||= error instanceof NotificationSettleError;
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
finally {
|
|
2592
|
+
if (!arrived) {
|
|
2593
|
+
barrier.leave();
|
|
2594
|
+
}
|
|
2595
|
+
release();
|
|
2596
|
+
}
|
|
2597
|
+
}));
|
|
2598
|
+
}
|
|
2599
|
+
return Promise.all(prepared.map(async (entry, index) => {
|
|
2600
|
+
const slot = state[index];
|
|
2601
|
+
// Every row failed before its provider call: the first failure is the
|
|
2602
|
+
// recipient's. A row whose answer could not be stored may have been
|
|
2603
|
+
// delivered, so it answers from the ledger, as `deliver` does. Every
|
|
2604
|
+
// other failure is reported here, as `runInline` reports it.
|
|
2605
|
+
const errors = slot?.errors ?? [];
|
|
2606
|
+
const thrown = slot?.callerError
|
|
2607
|
+
?? (errors.length > 0 && errors.length === entry.rows.length && slot?.settleFailed !== true
|
|
2608
|
+
? errors[0]?.error
|
|
2609
|
+
: undefined);
|
|
2610
|
+
for (const failure of errors) {
|
|
2611
|
+
if (failure.error !== thrown) {
|
|
2612
|
+
report(failure.error, 'dispatch', { sendKey: entry.key, provider: failure.provider, rowKey: failure.rowKey });
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
if (thrown !== undefined) {
|
|
2616
|
+
return { error: thrown };
|
|
2617
|
+
}
|
|
2618
|
+
let settled;
|
|
2619
|
+
try {
|
|
2620
|
+
settled = await settle(entry.key, slot?.refusals ?? []);
|
|
2621
|
+
}
|
|
2622
|
+
catch (error) {
|
|
2623
|
+
// The read failed, not the send: this recipient alone answers it.
|
|
2624
|
+
return { error };
|
|
2625
|
+
}
|
|
2626
|
+
return {
|
|
2627
|
+
outcome: entry.wasDelivered && settled.status === 'delivered'
|
|
2628
|
+
? { status: 'delivered', alreadyProcessed: true }
|
|
2629
|
+
: settled,
|
|
2630
|
+
};
|
|
2631
|
+
}));
|
|
2632
|
+
};
|
|
2633
|
+
const deliverBulk = async (raw, options = {}) => {
|
|
2634
|
+
const { input: bulk, keys } = parseNotificationBulkRequest(raw);
|
|
2635
|
+
refuseNoticeTopic(bulk.topic);
|
|
2636
|
+
const provider = registry.provider(bulk.provider);
|
|
2637
|
+
if (provider === undefined) {
|
|
2638
|
+
throw invalidInput(`This backend has no notifications provider ${bulk.provider}.`);
|
|
2639
|
+
}
|
|
2640
|
+
const capability = provider.delivery.bulk;
|
|
2641
|
+
const batchSize = Math.min(bulk.batchSize ?? capability?.maxRecipients ?? defaultBulkBatchSize, capability?.maxRecipients ?? maxNotificationRowsPerBulkBatch);
|
|
2642
|
+
const budgetMs = typeof options.budgetMs === 'number' && Number.isFinite(options.budgetMs) && options.budgetMs > 0
|
|
2643
|
+
? options.budgetMs
|
|
2644
|
+
: defaultSchedulerBudgetMs;
|
|
2645
|
+
const work = {
|
|
2646
|
+
left: typeof options.maxRecipients === 'number' && Number.isFinite(options.maxRecipients)
|
|
2647
|
+
? Math.max(0, Math.floor(options.maxRecipients))
|
|
2648
|
+
: Number.POSITIVE_INFINITY,
|
|
2649
|
+
};
|
|
2650
|
+
const lookups = {
|
|
2651
|
+
left: typeof options.maxLookups === 'number' && Number.isFinite(options.maxLookups)
|
|
2652
|
+
? Math.max(0, Math.floor(options.maxLookups))
|
|
2653
|
+
: Number.POSITIVE_INFINITY,
|
|
2654
|
+
};
|
|
2655
|
+
const elapsed = input.elapsedMs ?? (() => globalThis.performance.now());
|
|
2656
|
+
const startedAt = elapsed();
|
|
2657
|
+
// Recipients are prepared concurrently: their takes are made one at a
|
|
2658
|
+
// time, so the budget is read once and never spent twice.
|
|
2659
|
+
const budgets = paceBudgets();
|
|
2660
|
+
let paced = Promise.resolve();
|
|
2661
|
+
const pace = {
|
|
2662
|
+
take(provider) {
|
|
2663
|
+
const taken = paced.then(() => budgets.take(provider));
|
|
2664
|
+
paced = taken.catch(() => undefined);
|
|
2665
|
+
return taken;
|
|
2666
|
+
},
|
|
2667
|
+
refund: (provider) => budgets.refund(provider),
|
|
2668
|
+
};
|
|
2669
|
+
const calls = { batch: 0, single: 0 };
|
|
2670
|
+
// One copy answer per locale group for the whole call, shared by
|
|
2671
|
+
// planning, the provider calls, and the flush-time gate checks. It is
|
|
2672
|
+
// resolved once, when the first recipient needs a send.
|
|
2673
|
+
const copies = new Map();
|
|
2674
|
+
const lookupLimit = lookups.left;
|
|
2675
|
+
let primed;
|
|
2676
|
+
const ensureCopies = () => primed ??= primeBulkCopies(provider, {
|
|
2677
|
+
topic: bulk.topic,
|
|
2678
|
+
idempotencyKey: bulk.idempotencyKey,
|
|
2679
|
+
...(bulk.templateData === undefined ? {} : { templateData: bulk.templateData }),
|
|
2680
|
+
},
|
|
2681
|
+
// Only well-formed locales of recipients within the read budget: a
|
|
2682
|
+
// malformed one fails its own recipient's request check, never the
|
|
2683
|
+
// whole call through the resolver.
|
|
2684
|
+
bulk.recipients
|
|
2685
|
+
.slice(0, Number.isFinite(lookupLimit) ? lookupLimit : undefined)
|
|
2686
|
+
.filter((recipient) => recipient.templateData === undefined)
|
|
2687
|
+
.map((recipient) => recipient.locale ?? bulk.locale)
|
|
2688
|
+
.filter((locale) => isBoundedText(locale)), copies);
|
|
2689
|
+
const answers = keys.map(() => undefined);
|
|
2690
|
+
const shared = {
|
|
2691
|
+
...(bulk.templateData === undefined ? {} : { templateData: bulk.templateData }),
|
|
2692
|
+
...(bulk.data === undefined ? {} : { data: bulk.data }),
|
|
2693
|
+
...(bulk.locale === undefined ? {} : { locale: bulk.locale }),
|
|
2694
|
+
...(bulk.link === undefined ? {} : { link: bulk.link }),
|
|
2695
|
+
};
|
|
2696
|
+
for (let start = 0; start < keys.length; start += batchSize) {
|
|
2697
|
+
const end = Math.min(start + batchSize, keys.length);
|
|
2698
|
+
if (start > 0 && elapsed() - startedAt >= budgetMs) {
|
|
2699
|
+
for (let index = start; index < keys.length; index += 1) {
|
|
2700
|
+
answers[index] = { notAttempted: 'time-budget' };
|
|
2701
|
+
}
|
|
2702
|
+
break;
|
|
2703
|
+
}
|
|
2704
|
+
const ready = [];
|
|
2705
|
+
const limiter = createNotificationLimiter(bulkConcurrency);
|
|
2706
|
+
await Promise.all(bulk.recipients.slice(start, end).map(async (recipient, offset) => {
|
|
2707
|
+
const index = start + offset;
|
|
2708
|
+
// Every recipient's key is read once; past the read budget, none is.
|
|
2709
|
+
if (lookups.left <= 0) {
|
|
2710
|
+
answers[index] = { notAttempted: 'count-budget' };
|
|
2711
|
+
return;
|
|
2712
|
+
}
|
|
2713
|
+
lookups.left -= 1;
|
|
2714
|
+
await limiter.acquire();
|
|
2715
|
+
try {
|
|
2716
|
+
const made = await prepareBulkRecipient({
|
|
2717
|
+
...shared,
|
|
2718
|
+
...recipient,
|
|
2719
|
+
idempotencyKey: keys[index] ?? '',
|
|
2720
|
+
topic: bulk.topic,
|
|
2721
|
+
provider: bulk.provider,
|
|
2722
|
+
}, bulk.provider, pace, work, copies, ensureCopies);
|
|
2723
|
+
if ('done' in made) {
|
|
2724
|
+
answers[index] = made.done;
|
|
2725
|
+
}
|
|
2726
|
+
else if (made.rows.length === 0) {
|
|
2727
|
+
answers[index] = { outcome: await settle(made.key, [...made.refusals]) };
|
|
2728
|
+
}
|
|
2729
|
+
else {
|
|
2730
|
+
ready.push({ index, prepared: made });
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
catch (error) {
|
|
2734
|
+
answers[index] = { error };
|
|
2735
|
+
}
|
|
2736
|
+
finally {
|
|
2737
|
+
limiter.release();
|
|
2738
|
+
}
|
|
2739
|
+
}));
|
|
2740
|
+
ready.sort((left, right) => left.index - right.index);
|
|
2741
|
+
const ran = await runBulkBatch(provider, ready.map((entry) => entry.prepared), calls, copies);
|
|
2742
|
+
ready.forEach((entry, position) => {
|
|
2743
|
+
answers[entry.index] = ran[position] ?? { error: new Error('The batch answered no result.') };
|
|
2744
|
+
});
|
|
2745
|
+
}
|
|
2746
|
+
const recipients = bulk.recipients.map((recipient, index) => {
|
|
2747
|
+
const base = { player: recipient.player, idempotencyKey: keys[index] ?? '' };
|
|
2748
|
+
const answer = answers[index] ?? { notAttempted: 'time-budget' };
|
|
2749
|
+
if ('outcome' in answer) {
|
|
2750
|
+
return { ...base, status: 'answered', outcome: answer.outcome };
|
|
2751
|
+
}
|
|
2752
|
+
if ('notAttempted' in answer) {
|
|
2753
|
+
return { ...base, status: 'not-attempted', reason: answer.notAttempted };
|
|
2754
|
+
}
|
|
2755
|
+
const { error } = answer;
|
|
2756
|
+
if (!(error instanceof NotificationError)) {
|
|
2757
|
+
report(error, 'deliverBulk', base.idempotencyKey);
|
|
2758
|
+
}
|
|
2759
|
+
return {
|
|
2760
|
+
...base,
|
|
2761
|
+
status: 'failed',
|
|
2762
|
+
error: error instanceof NotificationError
|
|
2763
|
+
? { code: error.code, message: error.message, retryable: error.retryable }
|
|
2764
|
+
: {
|
|
2765
|
+
code: 'NOTIFICATIONS_TEMPORARILY_UNAVAILABLE',
|
|
2766
|
+
message: 'The send failed before an answer could be stored; retry the bulk send.',
|
|
2767
|
+
retryable: true,
|
|
2768
|
+
},
|
|
2769
|
+
};
|
|
2770
|
+
});
|
|
2771
|
+
return { recipients, summary: bulkSummaryOf(recipients), providerCalls: calls };
|
|
2772
|
+
};
|
|
1889
2773
|
return {
|
|
1890
2774
|
providers: () => registry.snapshot(),
|
|
1891
2775
|
async recordConsent(session, record) {
|
|
@@ -2130,6 +3014,7 @@ export function createNotificationService(input) {
|
|
|
2130
3014
|
}
|
|
2131
3015
|
throw busy(key);
|
|
2132
3016
|
},
|
|
3017
|
+
deliverBulk,
|
|
2133
3018
|
async dispatch(job) {
|
|
2134
3019
|
if (!isPlainObject(job)
|
|
2135
3020
|
|| !isBoundedText(job.sendKey)
|
|
@@ -2281,7 +3166,7 @@ export function createNotificationService(input) {
|
|
|
2281
3166
|
}
|
|
2282
3167
|
// A placeholder the fire would throw on is refused now, before the
|
|
2283
3168
|
// previous reminder's send is stopped or this one is written.
|
|
2284
|
-
assertReminderCopyFills(declaring, reminder.topic, reminder.templateData, catalog);
|
|
3169
|
+
assertReminderCopyFills(declaring, reminder.topic, reminder.templateData, catalog, !customCopy);
|
|
2285
3170
|
// One reminder per key: setting it replaces the previous one and, before
|
|
2286
3171
|
// the write, stops the previous one's send that waits; a scheduler run
|
|
2287
3172
|
// that fires the previous one meanwhile then fails to mark it and stops
|
|
@@ -2369,14 +3254,17 @@ export function createNotificationService(input) {
|
|
|
2369
3254
|
const touched = new Map();
|
|
2370
3255
|
// One pace for the whole run: rows taken in an earlier round count.
|
|
2371
3256
|
const pace = paceBudgets();
|
|
3257
|
+
// One copy cache for the whole run: a reminder's first check and its
|
|
3258
|
+
// dispatch, and recipients that share a locale, share a resolver call.
|
|
3259
|
+
const copies = new Map();
|
|
2372
3260
|
let failed = 0;
|
|
2373
3261
|
for (;;) {
|
|
2374
3262
|
let more;
|
|
2375
3263
|
let progressed;
|
|
2376
3264
|
try {
|
|
2377
|
-
const entries = await runScheduleEntries(limit, touched);
|
|
3265
|
+
const entries = await runScheduleEntries(limit, touched, copies);
|
|
2378
3266
|
failed += entries.failed;
|
|
2379
|
-
const dispatched = await dispatchDueRows({ limit }, pace);
|
|
3267
|
+
const dispatched = await dispatchDueRows({ limit }, pace, copies);
|
|
2380
3268
|
failed += dispatched.failed;
|
|
2381
3269
|
for (const result of dispatched.results) {
|
|
2382
3270
|
touched.set(result.sendKey, result.outcome);
|
|
@@ -2425,7 +3313,9 @@ export function createNotificationService(input) {
|
|
|
2425
3313
|
* is `NOTIFICATIONS_INVALID_INPUT` now, not a reminder that is stored and
|
|
2426
3314
|
* then dropped when it fires.
|
|
2427
3315
|
*/
|
|
2428
|
-
function assertReminderCopyFills(providers, topicId, templateData, catalog
|
|
3316
|
+
function assertReminderCopyFills(providers, topicId, templateData, catalog,
|
|
3317
|
+
/** False under a project's copy resolver, whose text replaces the catalog's copy. */
|
|
3318
|
+
catalogCopy) {
|
|
2429
3319
|
const data = templateData ?? {};
|
|
2430
3320
|
for (const provider of providers) {
|
|
2431
3321
|
const hostTemplate = provider.entry.content === 'host-template';
|
|
@@ -2442,7 +3332,7 @@ function assertReminderCopyFills(providers, topicId, templateData, catalog) {
|
|
|
2442
3332
|
}
|
|
2443
3333
|
const texts = mapped
|
|
2444
3334
|
? [mappedTitle, mappedBody]
|
|
2445
|
-
: topic.copy === undefined
|
|
3335
|
+
: topic.copy === undefined || !catalogCopy
|
|
2446
3336
|
? []
|
|
2447
3337
|
: Object.values(topic.copy).flatMap((localeCopy) => [localeCopy.title, localeCopy.body]);
|
|
2448
3338
|
for (const text of texts) {
|
|
@@ -2760,6 +3650,14 @@ function backedOff(row, at) {
|
|
|
2760
3650
|
* permission-order test in service.test.ts covers both sides).
|
|
2761
3651
|
*/
|
|
2762
3652
|
const earlyGateReasons = new Set(['topic-unknown', 'channel-unsupported', 'purpose-unsupported', 'no-endpoint', 'link-missing']);
|
|
3653
|
+
/**
|
|
3654
|
+
* The copy the gate renders while it decides whether a recipient is allowed
|
|
3655
|
+
* at all; the resolver's answer replaces it before anything is sent, so the
|
|
3656
|
+
* copy resolver is asked only for recipients the gate would allow.
|
|
3657
|
+
*/
|
|
3658
|
+
const copyStandIn = Object.freeze({
|
|
3659
|
+
copy: Object.freeze({ title: 'copy', body: 'copy' }),
|
|
3660
|
+
});
|
|
2763
3661
|
/** Writes that must not fail a settled delivery are tried this often. */
|
|
2764
3662
|
const maxSideWriteAttempts = 3;
|
|
2765
3663
|
/** Runs `write` up to `maxSideWriteAttempts` times with a short pause; reports only the last failure and never throws. */
|
|
@@ -2796,6 +3694,8 @@ function deliveryAnswerOf(answer) {
|
|
|
2796
3694
|
: { status: 'accepted' };
|
|
2797
3695
|
case 'unregistered':
|
|
2798
3696
|
return { status: 'unregistered' };
|
|
3697
|
+
case 'not-sent':
|
|
3698
|
+
return { status: 'not-sent' };
|
|
2799
3699
|
case 'rate-limited':
|
|
2800
3700
|
return typeof answer.retryAfterSeconds === 'number'
|
|
2801
3701
|
? { status: 'rate-limited', retryAfterSeconds: answer.retryAfterSeconds }
|
|
@@ -2812,10 +3712,68 @@ function deliveryAnswerOf(answer) {
|
|
|
2812
3712
|
}
|
|
2813
3713
|
: { status: 'rejected', reason: answer.reason };
|
|
2814
3714
|
case 'ambiguous':
|
|
2815
|
-
return
|
|
2816
|
-
|
|
2817
|
-
|
|
3715
|
+
return {
|
|
3716
|
+
status: 'ambiguous',
|
|
3717
|
+
...(typeof answer.providerCode === 'string' ? { providerCode: answer.providerCode.slice(0, 64) } : {}),
|
|
3718
|
+
...(answer.unattributed === true ? { unattributed: true } : {}),
|
|
3719
|
+
};
|
|
2818
3720
|
default:
|
|
2819
3721
|
return undefined;
|
|
2820
3722
|
}
|
|
2821
3723
|
}
|
|
3724
|
+
/** Recipients prepared and run at once in a bulk batch (each one's ledger reads and writes). */
|
|
3725
|
+
const bulkConcurrency = 8;
|
|
3726
|
+
/**
|
|
3727
|
+
* How old a bulk recipient's gate inputs (consent, history, host permission)
|
|
3728
|
+
* may be when its call starts; an older one is replayed with fresh reads.
|
|
3729
|
+
*/
|
|
3730
|
+
const bulkGateFreshMs = 30_000;
|
|
3731
|
+
/** Recipients per batch for a provider that sends one by one. */
|
|
3732
|
+
const defaultBulkBatchSize = 100;
|
|
3733
|
+
/** The most recipients one bulk batch runs for a provider that sends one by one. */
|
|
3734
|
+
const maxNotificationRowsPerBulkBatch = 1_000;
|
|
3735
|
+
function bulkSummaryOf(recipients) {
|
|
3736
|
+
const summary = {
|
|
3737
|
+
delivered: 0,
|
|
3738
|
+
alreadyProcessed: 0,
|
|
3739
|
+
deferred: 0,
|
|
3740
|
+
inFlight: 0,
|
|
3741
|
+
suppressed: 0,
|
|
3742
|
+
unavailable: 0,
|
|
3743
|
+
notAttempted: 0,
|
|
3744
|
+
failed: 0,
|
|
3745
|
+
};
|
|
3746
|
+
for (const recipient of recipients) {
|
|
3747
|
+
if (recipient.status === 'not-attempted') {
|
|
3748
|
+
summary.notAttempted += 1;
|
|
3749
|
+
}
|
|
3750
|
+
else if (recipient.status === 'failed') {
|
|
3751
|
+
summary.failed += 1;
|
|
3752
|
+
}
|
|
3753
|
+
else {
|
|
3754
|
+
switch (recipient.outcome.status) {
|
|
3755
|
+
case 'delivered':
|
|
3756
|
+
if (recipient.outcome.alreadyProcessed) {
|
|
3757
|
+
summary.alreadyProcessed += 1;
|
|
3758
|
+
}
|
|
3759
|
+
else {
|
|
3760
|
+
summary.delivered += 1;
|
|
3761
|
+
}
|
|
3762
|
+
break;
|
|
3763
|
+
case 'deferred':
|
|
3764
|
+
summary.deferred += 1;
|
|
3765
|
+
break;
|
|
3766
|
+
case 'in-flight':
|
|
3767
|
+
summary.inFlight += 1;
|
|
3768
|
+
break;
|
|
3769
|
+
case 'suppressed':
|
|
3770
|
+
summary.suppressed += 1;
|
|
3771
|
+
break;
|
|
3772
|
+
case 'unavailable':
|
|
3773
|
+
summary.unavailable += 1;
|
|
3774
|
+
break;
|
|
3775
|
+
}
|
|
3776
|
+
}
|
|
3777
|
+
}
|
|
3778
|
+
return summary;
|
|
3779
|
+
}
|