@mpgd/game-services 0.33.1 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
- import { assertNotificationConsentRecord, assertNotificationDeployment, assertNotificationEndpointRegistration, assertRecordNotificationNoticeInput, createNotificationNoticeIdempotencyKey, createNotificationReminderIdempotencyKey, evaluateNotificationDelivery, evaluateNotificationReminderConsent, fillNotificationTemplate, isNotificationTimestamp, isSameNotificationNotice, isValidNotificationTimeZone, latestNotificationConsent, listNotificationNotices, normalizeNotificationTimestamp, NotificationError, notificationKeyIssue, notificationNoticeCopyIssues, notificationNoticeSendFrom, notificationNoticeTopic, notificationProviderCarriesNotices, notificationReservedKeyPrefix, notificationSendFingerprint, notificationShellRenderedAdvertisingTargets, notificationTemplateDataIssue, toSeoulCalendarDate, } from '@mpgd/platform/notifications';
1
+ import { assertNotificationConsentRecord, assertNotificationDeployment, assertNotificationEndpointRegistration, assertRecordNotificationNoticeInput, createNotificationNoticeIdempotencyKey, createNotificationReminderIdempotencyKey, evaluateNotificationDelivery, evaluateNotificationReminderConsent, fillNotificationTemplate, isNotificationTimestamp, isSameNotificationNotice, isValidNotificationTimeZone, latestNotificationConsent, listNotificationNotices, normalizeNotificationTimestamp, notificationContentSource, NotificationError, notificationKeyIssue, notificationNoticeCopyIssues, notificationNoticeSendFrom, notificationNoticeTopic, notificationProviderCarriesNotices, notificationReservedKeyPrefix, notificationSendFingerprint, notificationShellRenderedAdvertisingTargets, notificationTemplateDataIssue, toSeoulCalendarDate, } from '@mpgd/platform/notifications';
2
+ import { createNotificationBatchBarrier, createNotificationLimiter, isInsideNotificationHoldWindow, parseNotificationBulkRequest, } from './bulk.js';
3
+ import { resolveCatalogNotificationCopy, resolveNotificationCopy, resolveNotificationCopyByLocale, } from './copy.js';
2
4
  import { fcmInstallationIdRefusal, notificationExpiredProviderCode, } from './delivery-provider.js';
3
5
  import { notificationHistoryRetentionDays, } from './registry.js';
4
6
  import { bestNotificationOutcome, formatNotificationInstant, isNotificationRowAbandoned, isNotificationRowDue, notificationAmbiguousReplayHours, notificationSendOutcome, } from './store.js';
@@ -118,6 +120,20 @@ export class NotificationConsentRateLimitError extends NotificationError {
118
120
  class NotificationSendInputError extends Error {
119
121
  name = 'NotificationSendInputError';
120
122
  }
123
+ /**
124
+ * The copy resolver could not answer. Retryable, like a host that cannot
125
+ * answer, but a send's first check fails as a whole on it: copy is resolved
126
+ * per locale, so skipping only the recipients of one locale would store a
127
+ * send that never reaches them.
128
+ */
129
+ class NotificationCopyUnavailableError extends NotificationError {
130
+ constructor(topic, options) {
131
+ super('NOTIFICATIONS_TEMPORARILY_UNAVAILABLE', {
132
+ message: `The copy resolver could not answer for topic ${topic}.`,
133
+ cause: options.cause,
134
+ });
135
+ }
136
+ }
121
137
  /**
122
138
  * The provider accepted or may have sent, but storing its answer failed. It
123
139
  * is retryable for the caller, who answers from the ledger by the same key;
@@ -132,6 +148,23 @@ class NotificationSettleError extends NotificationError {
132
148
  });
133
149
  }
134
150
  }
151
+ /** The cache key of one resolver answer: topic, provider, target, locale, notice variant, and template data. */
152
+ function copyCacheKey(topic, provider, target, locale, origin, templateData) {
153
+ return JSON.stringify([
154
+ topic,
155
+ provider,
156
+ target ?? null,
157
+ locale ?? null,
158
+ origin.kind === 'notice' ? origin.variant : null,
159
+ templateData === undefined
160
+ ? null
161
+ : Object.entries(templateData).sort(([left], [right]) => (left < right
162
+ ? -1
163
+ : left > right
164
+ ? 1
165
+ : 0)),
166
+ ]);
167
+ }
135
168
  export function createNotificationService(input) {
136
169
  const { store, registry, catalog } = input;
137
170
  const deployment = input.deployment === undefined
@@ -147,6 +180,13 @@ export function createNotificationService(input) {
147
180
  /** The claim lease after `attempts` ambiguous attempts: doubling, up to an hour (or the lease, if longer). */
148
181
  const ambiguousLeaseMs = (attempts) => Math.min(leaseMs * 2 ** Math.min(attempts, 16), Math.max(leaseMs, maxAmbiguousLeaseMs));
149
182
  const random = input.random ?? Math.random;
183
+ const resolveCopy = input.resolveCopy ?? resolveCatalogNotificationCopy;
184
+ /**
185
+ * A project's resolver answers text the backend cannot check ahead of the
186
+ * send: its placeholders are checked against the send's data when it
187
+ * answers, and a reminder's set time checks no catalog copy.
188
+ */
189
+ const customCopy = input.resolveCopy !== undefined;
150
190
  const noticeTopic = catalog.notices?.topic;
151
191
  if (catalog.notices !== undefined) {
152
192
  const issues = notificationNoticeCopyIssues(catalog.notices.copy, catalog.defaultLocale);
@@ -198,6 +238,131 @@ export function createNotificationService(input) {
198
238
  }
199
239
  return undefined;
200
240
  };
241
+ /**
242
+ * The copy the resolver answers for a recipient's send. It is asked only
243
+ * once the gate, deciding with `copyStandIn`, allowed the recipient with
244
+ * content from copy (`notificationContentSource` is `catalog`), so the
245
+ * stand-in never reaches a provider.
246
+ *
247
+ * `cache` lives for one invocation (one `deliver`, one queued job, one
248
+ * `dispatchDue`, or one scheduler run) and keeps one answer per topic,
249
+ * provider, target, locale, notice variant, and template data, so the
250
+ * recipients and rows that share them share one resolver call, and a
251
+ * send's first check and its provider call in the same invocation do too.
252
+ * The template data is part of the key because the resolver is given it.
253
+ *
254
+ * A `NotificationError` the resolver throws keeps its own code and passes
255
+ * through (the Worker's cron budget does); any other throw is a copy
256
+ * outage (`NotificationCopyUnavailableError`). An answer that cannot be
257
+ * sent is reported and is no copy.
258
+ */
259
+ const copyFor = async (recipient, request, origin, cache) => {
260
+ const topic = topicFor(recipient.provider, recipient.target, request.topic, origin);
261
+ if (topic === undefined) {
262
+ return {};
263
+ }
264
+ const provider = recipient.provider.entry.provider;
265
+ const locale = request.locale ?? recipient.locale;
266
+ const key = copyCacheKey(topic.id, provider, recipient.target, locale, origin, request.templateData);
267
+ let resolved = cache.get(key);
268
+ if (resolved === undefined) {
269
+ resolved = (async () => {
270
+ let resolution;
271
+ try {
272
+ resolution = await resolveNotificationCopy(resolveCopy, {
273
+ topic,
274
+ provider,
275
+ ...(locale === undefined ? {} : { locale }),
276
+ defaultLocale: catalog.defaultLocale,
277
+ ...(request.templateData === undefined ? {} : { templateData: request.templateData }),
278
+ ...(origin.kind === 'notice' ? { notice: { variant: origin.variant } } : {}),
279
+ });
280
+ }
281
+ catch (error) {
282
+ if (error instanceof NotificationError) {
283
+ throw error;
284
+ }
285
+ throw new NotificationCopyUnavailableError(topic.id, { cause: error });
286
+ }
287
+ return checkedCopy(resolution, topic.id, request.idempotencyKey, provider, locale, request.templateData);
288
+ })();
289
+ cache.set(key, resolved);
290
+ }
291
+ return resolved;
292
+ };
293
+ /** A resolver's answer as the gate takes it: no copy (reported) when it is missing or cannot be filled. */
294
+ const checkedCopy = (resolution, topicId, sendKey, provider, locale, templateData) => {
295
+ const where = { sendKey, provider };
296
+ if (resolution.copy === undefined) {
297
+ // Reported, so a missing translation is not only a quiet copy-missing.
298
+ report(new TypeError(`Topic ${topicId}: ${resolution.issue ?? `no copy for ${locale ?? catalog.defaultLocale}`}.`), 'copy.resolve', where);
299
+ return {};
300
+ }
301
+ if (customCopy) {
302
+ try {
303
+ fillNotificationTemplate(resolution.copy.title, templateData);
304
+ fillNotificationTemplate(resolution.copy.body, templateData);
305
+ }
306
+ catch (error) {
307
+ // The project's text needs a value this send does not carry.
308
+ report(new TypeError(`Topic ${topicId}: ${describe(error)}`), 'copy.resolve', where);
309
+ return {};
310
+ }
311
+ }
312
+ return { copy: resolution.copy };
313
+ };
314
+ /**
315
+ * A bulk send's copy, up front (BK1): one `resolveNotificationCopyByLocale`
316
+ * call for the locales its recipients name (the send's `locale`, or each
317
+ * recipient's own), stored in the invocation's `cache` under the keys
318
+ * `copyFor` reads, so planning, the provider call, and the flush-time
319
+ * gate check share each locale group's one answer. A recipient whose
320
+ * locale comes from its registration joins its group through the same
321
+ * cache. Only for a platform-user provider whose content comes from copy
322
+ * (a host template or a mapping's own text asks no resolver). It runs
323
+ * when the first recipient needs a send (`ensureCopies` in `deliverBulk`),
324
+ * so a retry whose recipients were all sent never asks the resolver. A
325
+ * resolver that throws here is reported and caches nothing: each locale
326
+ * group is then asked when a recipient the gate allows needs it, and a
327
+ * failure there fails that group's recipients, retryable.
328
+ */
329
+ const primeBulkCopies = async (provider, request, locales, cache) => {
330
+ if (provider.entry.addressing !== 'platform-user') {
331
+ return;
332
+ }
333
+ const origin = { kind: 'send' };
334
+ const topic = topicFor(provider, undefined, request.topic, origin);
335
+ if (topic === undefined || notificationContentSource(topic, provider.entry.content) !== 'catalog') {
336
+ return;
337
+ }
338
+ const id = provider.entry.provider;
339
+ const keyOf = (locale) => copyCacheKey(topic.id, id, undefined, locale, origin, request.templateData);
340
+ const wanted = [...new Set(locales)].filter((locale) => !cache.has(keyOf(locale)));
341
+ if (wanted.length === 0) {
342
+ return;
343
+ }
344
+ let resolved;
345
+ try {
346
+ resolved = await resolveNotificationCopyByLocale(resolveCopy, {
347
+ topic,
348
+ provider: id,
349
+ defaultLocale: catalog.defaultLocale,
350
+ ...(request.templateData === undefined ? {} : { templateData: request.templateData }),
351
+ locales: wanted,
352
+ });
353
+ }
354
+ catch (error) {
355
+ // The up-front answers are an optimization: a failure caches nothing,
356
+ // so each locale group is asked again only when a recipient the gate
357
+ // allows needs it (`copyFor`), and a locale of refused recipients never
358
+ // fails the others.
359
+ report(error, 'copy.resolve.bulk', { sendKey: request.idempotencyKey, provider: id });
360
+ return;
361
+ }
362
+ for (const locale of wanted) {
363
+ cache.set(keyOf(locale), Promise.resolve(checkedCopy(resolved.get(locale) ?? {}, topic.id, request.idempotencyKey, id, locale, request.templateData)));
364
+ }
365
+ };
201
366
  const refuseNoticeTopic = (topic) => {
202
367
  if (noticeTopic !== undefined && topic === noticeTopic) {
203
368
  throw invalidInput(`Topic ${topic} is reserved for consent notices, which only the scheduler sends.`);
@@ -415,7 +580,7 @@ export function createNotificationService(input) {
415
580
  ? early
416
581
  : undefined;
417
582
  };
418
- const decide = (request, origin, recipient, consent, history, serverPermission, at) => {
583
+ const decide = (request, origin, recipient, consent, history, serverPermission, at, resolvedCopy) => {
419
584
  const topic = topicFor(recipient.provider, recipient.target, request.topic, origin);
420
585
  const locale = request.locale ?? recipient.locale;
421
586
  try {
@@ -437,6 +602,7 @@ export function createNotificationService(input) {
437
602
  ...(locale === undefined ? {} : { locale }),
438
603
  defaultLocale: catalog.defaultLocale,
439
604
  ...(request.templateData === undefined ? {} : { templateData: request.templateData }),
605
+ ...(resolvedCopy === undefined ? {} : { resolvedCopy }),
440
606
  }),
441
607
  };
442
608
  }
@@ -488,7 +654,8 @@ export function createNotificationService(input) {
488
654
  state: { since: formatNotificationInstant(since), delaySeconds, attempts: attempts + 1 },
489
655
  };
490
656
  };
491
- const callProvider = async (recipient, request, topic, decision, at) => {
657
+ /** What the provider is asked to send for one allowed recipient. */
658
+ const deliveryRequestOf = (recipient, request, topic, decision, at) => {
492
659
  const mappedLink = topic.mapping?.['link'];
493
660
  const link = request.link ?? (typeof mappedLink === 'string' ? mappedLink : undefined);
494
661
  const target = recipient.endpoint !== undefined
@@ -499,36 +666,45 @@ export function createNotificationService(input) {
499
666
  }
500
667
  : { kind: 'platform-user', id: recipient.platformId ?? '' };
501
668
  const locale = request.locale ?? recipient.locale;
669
+ return {
670
+ idempotencyKey: recipient.rowKey,
671
+ sendKey: request.idempotencyKey,
672
+ provider: recipient.provider.entry.provider,
673
+ player: request.player,
674
+ topic,
675
+ recipient: target,
676
+ content: decision.content,
677
+ ...(decision.expiresAt === undefined ? {} : { expiresAt: decision.expiresAt }),
678
+ ...(request.templateData === undefined ? {} : { templateData: request.templateData }),
679
+ ...(request.data === undefined ? {} : { data: request.data }),
680
+ ...(link === undefined ? {} : { link }),
681
+ ...(locale === undefined ? {} : { locale }),
682
+ at: formatNotificationInstant(at),
683
+ };
684
+ };
685
+ /** One recipient's provider call (`send`), as one of the five answers. */
686
+ const sendOne = async (provider, delivery) => {
502
687
  try {
503
- const answer = await recipient.provider.delivery.send({
504
- idempotencyKey: recipient.rowKey,
505
- sendKey: request.idempotencyKey,
506
- provider: recipient.provider.entry.provider,
507
- player: request.player,
508
- topic,
509
- recipient: target,
510
- content: decision.content,
511
- ...(decision.expiresAt === undefined ? {} : { expiresAt: decision.expiresAt }),
512
- ...(request.templateData === undefined ? {} : { templateData: request.templateData }),
513
- ...(request.data === undefined ? {} : { data: request.data }),
514
- ...(link === undefined ? {} : { link }),
515
- ...(locale === undefined ? {} : { locale }),
516
- at: formatNotificationInstant(at),
517
- });
688
+ const answer = await provider.delivery.send(delivery);
518
689
  const known = deliveryAnswerOf(answer);
519
690
  if (known === undefined) {
520
691
  // A provider that broke the contract may still have sent it.
521
- report(new TypeError('The provider answered no known delivery answer.'), 'provider.send', request.idempotencyKey);
692
+ report(new TypeError('The provider answered no known delivery answer.'), 'provider.send', delivery.sendKey);
522
693
  return { status: 'ambiguous' };
523
694
  }
524
695
  return known;
525
696
  }
526
697
  catch (error) {
527
698
  // The provider may have sent it: one rule for every failure it cannot prove.
528
- report(error, 'provider.send', request.idempotencyKey, true);
699
+ report(error, 'provider.send', delivery.sendKey, true);
529
700
  return { status: 'ambiguous' };
530
701
  }
531
702
  };
703
+ const callProvider = (recipient, request, topic, decision, at,
704
+ // When the caller's claim on the row ends (epoch ms); a bulk batch checks it before its call.
705
+ _claimedUntil,
706
+ // The gate again at a later instant; a bulk batch runs it just before its call.
707
+ _regate) => sendOne(recipient.provider, deliveryRequestOf(recipient, request, topic, decision, at));
532
708
  /**
533
709
  * The rows a new send stores, after a first gate check per recipient:
534
710
  * allowed recipients become queued rows, deferred ones deferred rows, and
@@ -537,7 +713,7 @@ export function createNotificationService(input) {
537
713
  * at `queuedUntil` for a scheduler item, whose rows are due at once and
538
714
  * leave through the paced due path.
539
715
  */
540
- const plan = async (request, origin, at, queuedUntil) => {
716
+ const plan = async (request, origin, at, queuedUntil, copies = new Map()) => {
541
717
  const recipients = await recipientsOf(request, at, origin);
542
718
  if (recipients.length === 0) {
543
719
  return { rows: [], refusals: [noEndpoint()] };
@@ -557,20 +733,32 @@ export function createNotificationService(input) {
557
733
  perProvider.set(id, inputs);
558
734
  }
559
735
  const held = inputs;
560
- const evaluate = (permission) => decide(request, origin, recipient, consent, held.history, permission, at);
561
736
  // One read per provider and send, made only once a recipient needs it.
562
737
  // A host that cannot answer is skipped; the send fails only when no
563
- // provider could be checked and none passed.
738
+ // provider could be checked and none passed. The copy resolver is
739
+ // asked only for a recipient the gate would allow, once per target and
740
+ // locale. Its failure fails the whole check rather than skipping the
741
+ // recipients of one locale, which would store a send that never
742
+ // reaches them.
564
743
  let decision;
744
+ let resolvingCopy = false;
565
745
  try {
566
- const early = earlyRefusal(recipient.provider, evaluate);
567
- decision = early === undefined
568
- ? evaluate(await (held.permission ??= readServerPermission(recipient.provider, request))).decision
569
- : early.decision;
746
+ const evaluate = (permission, resolvedCopy = copyStandIn) => decide(request, origin, recipient, consent, held.history, permission, at, resolvedCopy);
747
+ const early = earlyRefusal(recipient.provider, (standIn) => evaluate(standIn));
748
+ const permission = early === undefined
749
+ ? await (held.permission ??= readServerPermission(recipient.provider, request))
750
+ : undefined;
751
+ decision = early?.decision ?? evaluate(permission).decision;
752
+ if (decision.status === 'allowed' && decision.content.source === 'catalog') {
753
+ resolvingCopy = true;
754
+ decision = evaluate(permission, await copyFor(recipient, request, origin, copies)).decision;
755
+ }
570
756
  checked += 1;
571
757
  }
572
758
  catch (error) {
573
- if (error instanceof NotificationError && error.code === 'NOTIFICATIONS_TEMPORARILY_UNAVAILABLE') {
759
+ if (!resolvingCopy
760
+ && error instanceof NotificationError
761
+ && error.code === 'NOTIFICATIONS_TEMPORARILY_UNAVAILABLE') {
574
762
  hostFailure ??= error;
575
763
  continue;
576
764
  }
@@ -686,7 +874,9 @@ export function createNotificationService(input) {
686
874
  * A notice the game recorded since the claim, or a reminder cancelled or
687
875
  * replaced since the claim, is dropped instead of sent.
688
876
  */
689
- const runRow = async (sendKey, provider, rowKey, mode) => {
877
+ const runRow = async (sendKey, provider, rowKey, mode, copies = new Map(),
878
+ // A bulk send gathers its recipients' calls into one provider call.
879
+ call = callProvider) => {
690
880
  let claimed;
691
881
  for (let attempt = 0; attempt < maxLedgerAttempts && claimed === undefined; attempt += 1) {
692
882
  const at = nowMs();
@@ -757,15 +947,35 @@ export function createNotificationService(input) {
757
947
  let evaluated;
758
948
  let recipient;
759
949
  let at;
950
+ // The same gate at a later instant, with the inputs read now: a bulk batch checks it again before its call.
951
+ let regate;
760
952
  try {
761
953
  recipient = await recipientOfRow(registered, send.request, row, claimed.at);
762
954
  const consent = await store.listConsent(send.request.player);
763
- const early = earlyRefusal(registered, (standIn) => decide(send.request, send.origin, recipient, consent, [], standIn, nowMs()));
955
+ const early = earlyRefusal(registered, (standIn) => decide(send.request, send.origin, recipient, consent, [], standIn, nowMs(), copyStandIn));
764
956
  const serverPermission = early === undefined
765
957
  ? await readServerPermission(registered, send.request)
766
958
  : undefined;
767
959
  at = nowMs();
768
- evaluated = early ?? decide(send.request, send.origin, recipient, consent, historyOn(await loadHistory(send.request.player, sendKey, at, { key: sendKey, claimedAt: claimed.at }), row.provider), serverPermission, at);
960
+ const history = early === undefined
961
+ ? historyOn(await loadHistory(send.request.player, sendKey, at, { key: sendKey, claimedAt: claimed.at }), row.provider)
962
+ : [];
963
+ const evaluate = (resolvedCopy) => decide(send.request, send.origin, recipient, consent, history, serverPermission, at, resolvedCopy);
964
+ evaluated = early ?? evaluate(copyStandIn);
965
+ let gatedCopy = copyStandIn;
966
+ // The copy resolver is asked only once the gate would allow the call.
967
+ if (early === undefined
968
+ && evaluated.decision.status === 'allowed'
969
+ && evaluated.decision.content.source === 'catalog') {
970
+ gatedCopy = await copyFor(recipient, send.request, send.origin, copies);
971
+ // The resolver took time: the gate decides again at the time of the
972
+ // call, so a window that opened meanwhile (the night window) holds it.
973
+ at = nowMs();
974
+ evaluated = evaluate(gatedCopy);
975
+ }
976
+ const gated = recipient;
977
+ const copy = gatedCopy;
978
+ regate = (when) => decide(send.request, send.origin, gated, consent, history, serverPermission, when, copy).decision;
769
979
  }
770
980
  catch (error) {
771
981
  if (error instanceof NotificationSendInputError) {
@@ -812,7 +1022,7 @@ export function createNotificationService(input) {
812
1022
  await complete(sendKey, row, token, { kind: 'remove' }, nowMs());
813
1023
  return {};
814
1024
  }
815
- const answer = await callProvider(recipient, send.request, topic, decision, at);
1025
+ const answer = await call(recipient, send.request, topic, decision, at, row.state.status === 'in-flight' ? Date.parse(row.state.leaseUntil) : undefined, regate);
816
1026
  const answeredAt = nowMs();
817
1027
  switch (answer.status) {
818
1028
  case 'accepted':
@@ -839,7 +1049,10 @@ export function createNotificationService(input) {
839
1049
  const leaseUntil = answeredAt + ambiguousLeaseMs(ambiguousAttempts);
840
1050
  // The replay window started at the claim that reached the provider (or an earlier one).
841
1051
  const firstAttemptedAt = row.firstAttemptedAt ?? formatNotificationInstant(at);
842
- if (isNotificationRowAbandoned({ ...row, firstAttemptedAt }, leaseUntil)) {
1052
+ if (answer.unattributed === true) {
1053
+ report(new Error(`A bulk call reached some of its recipients without naming them${answer.providerCode === undefined ? '' : ` (${answer.providerCode})`}: the delivery stays in flight and is not replayed, so no recipient gets it twice.`), 'dispatch.unattributed', { sendKey, provider: row.provider, rowKey: row.key });
1054
+ }
1055
+ else if (isNotificationRowAbandoned({ ...row, firstAttemptedAt }, leaseUntil)) {
843
1056
  report(new Error(answer.providerCode === undefined
844
1057
  ? 'Ambiguous answers for a day: the delivery stays in flight and is not replayed again.'
845
1058
  : `Ambiguous answers for a day (${answer.providerCode}): the delivery stays in flight and is not replayed again.`), 'dispatch.abandoned', { sendKey, provider: row.provider, rowKey: row.key });
@@ -851,6 +1064,7 @@ export function createNotificationService(input) {
851
1064
  ...backoff,
852
1065
  firstAttemptedAt,
853
1066
  ...(ambiguousAttempts === 0 ? {} : { ambiguousAttempts }),
1067
+ ...(answer.unattributed === true ? { unattributed: true } : {}),
854
1068
  state: {
855
1069
  status: 'in-flight',
856
1070
  leaseUntil: formatNotificationInstant(leaseUntil),
@@ -885,6 +1099,12 @@ export function createNotificationService(input) {
885
1099
  }
886
1100
  await complete(sendKey, row, token, { kind: 'remove' }, answeredAt);
887
1101
  return { refusal: { status: 'unavailable', reason: 'provider-rejected' } };
1102
+ case 'not-sent':
1103
+ // No call was made for it (a bulk batch left it out): the row goes
1104
+ // back as it was before this claim, due again shortly. It is not a
1105
+ // rate limit, so it neither backs off nor nears the 24-hour give-up.
1106
+ await complete(sendKey, row, token, { kind: 'set', row: backedOff(before, answeredAt) }, answeredAt);
1107
+ return {};
888
1108
  case 'rate-limited': {
889
1109
  // Nothing was sent: the claim is released and the row waits.
890
1110
  const next = nextRateLimit(row.rateLimit, answer.retryAfterSeconds, answeredAt);
@@ -957,13 +1177,13 @@ export function createNotificationService(input) {
957
1177
  * got anywhere, so a caller never mistakes a send that did nothing for an
958
1178
  * ambiguous one.
959
1179
  */
960
- const runInline = async (sendKey, rows, mode) => {
1180
+ const runInline = async (sendKey, rows, mode, copies = new Map()) => {
961
1181
  const refusals = [];
962
1182
  let callerError;
963
1183
  const errors = [];
964
1184
  for (const row of rows) {
965
1185
  try {
966
- const { refusal } = await runRow(sendKey, row.provider, row.key, mode);
1186
+ const { refusal } = await runRow(sendKey, row.provider, row.key, mode, copies);
967
1187
  if (refusal !== undefined) {
968
1188
  refusals.push(refusal);
969
1189
  }
@@ -996,9 +1216,11 @@ export function createNotificationService(input) {
996
1216
  };
997
1217
  /** Evaluates a send this call created, holding its first-evaluation claim. */
998
1218
  const sendNew = async (send, version, at) => {
1219
+ // One copy cache for the first check and the inline provider calls.
1220
+ const copies = new Map();
999
1221
  let planned;
1000
1222
  try {
1001
- planned = await plan(send.request, send.origin, at);
1223
+ planned = await plan(send.request, send.origin, at, undefined, copies);
1002
1224
  }
1003
1225
  catch (error) {
1004
1226
  // Nothing was sent: the key is released so a retry evaluates again.
@@ -1039,7 +1261,7 @@ export function createNotificationService(input) {
1039
1261
  }
1040
1262
  return notificationSendOutcome(stored) ?? { status: 'in-flight' };
1041
1263
  }
1042
- const ran = await runInline(send.key, queued, 'job');
1264
+ const ran = await runInline(send.key, queued, 'job', copies);
1043
1265
  return settle(send.key, [...refusals, ...ran.refusals]);
1044
1266
  };
1045
1267
  // Due rows (the scheduler's dispatch).
@@ -1107,6 +1329,13 @@ export function createNotificationService(input) {
1107
1329
  budgets.set(provider, left - 1);
1108
1330
  return left > 0;
1109
1331
  },
1332
+ /** Gives back one `take` that made no delivery after all. */
1333
+ refund(provider) {
1334
+ const left = budgets.get(provider);
1335
+ if (left !== undefined) {
1336
+ budgets.set(provider, left + 1);
1337
+ }
1338
+ },
1110
1339
  };
1111
1340
  };
1112
1341
  /**
@@ -1116,13 +1345,18 @@ export function createNotificationService(input) {
1116
1345
  * Answers how many sends were listed and how many of them the pace held
1117
1346
  * back entirely.
1118
1347
  */
1119
- const dispatchDueRows = async (options = {}, pace = paceBudgets()) => {
1348
+ const dispatchDueRows = async (options = {}, pace = paceBudgets(), copies = new Map()) => {
1120
1349
  const limit = batchLimit(options.limit);
1121
1350
  const keys = await store.listDueSends({ at: formatNotificationInstant(nowMs()), limit });
1122
1351
  const listed = keys.length;
1123
- const results = [];
1352
+ // Each listed send's result, in listing order; a failed one has none.
1353
+ const slots = new Map();
1124
1354
  let failed = 0;
1125
1355
  let paced = 0;
1356
+ // Sends whose due rows all go to one provider with a bulk path: replayed
1357
+ // together through its `sendBatch` (BK1), each row still its own claim,
1358
+ // gate check, and answer.
1359
+ const gathered = new Map();
1126
1360
  for (const sendKey of keys) {
1127
1361
  try {
1128
1362
  const held = await store.getSend(sendKey);
@@ -1160,20 +1394,52 @@ export function createNotificationService(input) {
1160
1394
  failed += 1;
1161
1395
  report(error, 'queue.enqueue', sendKey);
1162
1396
  }
1163
- results.push({ sendKey, outcome: await settle(sendKey, []) });
1397
+ slots.set(sendKey, { sendKey, outcome: await settle(sendKey, []) });
1164
1398
  continue;
1165
1399
  }
1166
- const ran = await runInline(sendKey, due, 'due');
1400
+ const only = due.every((row) => row.provider === due[0]?.provider)
1401
+ ? registry.provider(due[0]?.provider ?? '')
1402
+ : undefined;
1403
+ if (only?.delivery.bulk !== undefined && only.delivery.sendBatch !== undefined) {
1404
+ const group = gathered.get(only.entry.provider)
1405
+ ?? { provider: only, sends: [] };
1406
+ group.sends.push({ key: sendKey, rows: due });
1407
+ gathered.set(only.entry.provider, group);
1408
+ slots.set(sendKey, undefined);
1409
+ continue;
1410
+ }
1411
+ const ran = await runInline(sendKey, due, 'due', copies);
1167
1412
  if (ran.failed > 0) {
1168
1413
  failed += 1;
1169
1414
  }
1170
- results.push({ sendKey, outcome: await settle(sendKey, ran.refusals) });
1415
+ slots.set(sendKey, { sendKey, outcome: await settle(sendKey, ran.refusals) });
1171
1416
  }
1172
1417
  catch (error) {
1173
1418
  failed += 1;
1174
1419
  report(error, 'dispatchDue', sendKey);
1175
1420
  }
1176
1421
  }
1422
+ for (const { provider, sends } of gathered.values()) {
1423
+ let answers;
1424
+ try {
1425
+ answers = await runBulkBatch(provider, sends.map((send) => ({ key: send.key, rows: send.rows, mode: 'due', refusals: [], wasDelivered: false })), { batch: 0, single: 0 }, copies);
1426
+ }
1427
+ catch (error) {
1428
+ failed += sends.length;
1429
+ report(error, 'dispatchDue', sends[0]?.key ?? '');
1430
+ continue;
1431
+ }
1432
+ sends.forEach((send, index) => {
1433
+ const answer = answers[index];
1434
+ if (answer !== undefined && 'outcome' in answer) {
1435
+ slots.set(send.key, { sendKey: send.key, outcome: answer.outcome });
1436
+ return;
1437
+ }
1438
+ failed += 1;
1439
+ report(answer !== undefined && 'error' in answer ? answer.error : new Error('No answer.'), 'dispatchDue', send.key);
1440
+ });
1441
+ }
1442
+ const results = [...slots.values()].filter((result) => result !== undefined);
1177
1443
  return { results, failed, listed, paced };
1178
1444
  };
1179
1445
  // Scheduler items.
@@ -1231,11 +1497,11 @@ export function createNotificationService(input) {
1231
1497
  * releases the key and throws `NOTIFICATIONS_INVALID_INPUT`; any other
1232
1498
  * failure releases it and rethrows.
1233
1499
  */
1234
- const storeItemRows = async (created, at, dueAt) => {
1500
+ const storeItemRows = async (created, at, dueAt, copies = new Map()) => {
1235
1501
  const { send, version } = created;
1236
1502
  let planned;
1237
1503
  try {
1238
- planned = await plan(send.request, send.origin, at, Math.min(dueAt, at));
1504
+ planned = await plan(send.request, send.origin, at, Math.min(dueAt, at), copies);
1239
1505
  }
1240
1506
  catch (error) {
1241
1507
  await releaseQuietly(send.key, version);
@@ -1372,7 +1638,7 @@ export function createNotificationService(input) {
1372
1638
  * Answers `busy` while another run is making the send, and `stale` when
1373
1639
  * this run read a reminder a set has replaced since: neither worked it.
1374
1640
  */
1375
- const fireReminder = async (held, reminder) => {
1641
+ const fireReminder = async (held, reminder, copies = new Map()) => {
1376
1642
  const { player } = held.entry;
1377
1643
  const at = nowMs();
1378
1644
  let key;
@@ -1445,7 +1711,7 @@ export function createNotificationService(input) {
1445
1711
  // The rows are due when the entry was: a past fireAt counts from its
1446
1712
  // set time. A pending reminder always has a due time (runScheduler
1447
1713
  // takes only entries with one); fireAt only satisfies the type.
1448
- result = await storeItemRows(made.created, at, Date.parse(held.entry.dueAt ?? reminder.fireAt));
1714
+ result = await storeItemRows(made.created, at, Date.parse(held.entry.dueAt ?? reminder.fireAt), copies);
1449
1715
  }
1450
1716
  catch (error) {
1451
1717
  if (error instanceof NotificationError && error.code === 'NOTIFICATIONS_INVALID_INPUT') {
@@ -1538,7 +1804,7 @@ export function createNotificationService(input) {
1538
1804
  * backoff after a refused push (an hour, doubling to a day). With nothing
1539
1805
  * owed it is removed.
1540
1806
  */
1541
- const checkNotices = async (held, check) => {
1807
+ const checkNotices = async (held, check, copies = new Map()) => {
1542
1808
  const { player } = held.entry;
1543
1809
  if (!carriesNotices() || noticeTopic === undefined) {
1544
1810
  await removeEntry(held);
@@ -1613,7 +1879,7 @@ export function createNotificationService(input) {
1613
1879
  continue;
1614
1880
  }
1615
1881
  try {
1616
- const result = await storeItemRows(made.created, at, seoulMidnight(sendFrom));
1882
+ const result = await storeItemRows(made.created, at, seoulMidnight(sendFrom), copies);
1617
1883
  if (result === undefined) {
1618
1884
  // Another run took the key over: its push is looked at again.
1619
1885
  waiting = true;
@@ -1803,7 +2069,7 @@ export function createNotificationService(input) {
1803
2069
  * checked, or forgotten, with each send it touched. A schedule that cannot
1804
2070
  * be read is counted and reported, and the round's due rows still go out.
1805
2071
  */
1806
- const runScheduleEntries = async (limit, touched) => {
2072
+ const runScheduleEntries = async (limit, touched, copies = new Map()) => {
1807
2073
  let failed = 0;
1808
2074
  let due = [];
1809
2075
  try {
@@ -1824,7 +2090,7 @@ export function createNotificationService(input) {
1824
2090
  }
1825
2091
  const { item } = held.entry;
1826
2092
  if (item.kind === 'notices') {
1827
- for (const result of await checkNotices(held, item)) {
2093
+ for (const result of await checkNotices(held, item, copies)) {
1828
2094
  touched.set(result.sendKey, result.outcome);
1829
2095
  }
1830
2096
  }
@@ -1832,7 +2098,7 @@ export function createNotificationService(input) {
1832
2098
  await forgetFiredReminder(held, item);
1833
2099
  }
1834
2100
  else {
1835
- const result = await fireReminder(held, item);
2101
+ const result = await fireReminder(held, item, copies);
1836
2102
  if (result === 'busy') {
1837
2103
  // Another run holds its send, or a live claim of other content
1838
2104
  // still occupies the key: it waits a minute, so the next round
@@ -1886,6 +2152,624 @@ export function createNotificationService(input) {
1886
2152
  }
1887
2153
  return current;
1888
2154
  };
2155
+ // Bulk sends (RFC open question 5, BK1).
2156
+ /**
2157
+ * One recipient of a bulk send, made ready the way `deliver` makes a
2158
+ * send: answered from its key's state, or created, planned through the
2159
+ * gate, and stored with the rows to run now.
2160
+ */
2161
+ const prepareBulkRecipient = async (raw, provider, pace,
2162
+ // Recipients this call may still attempt; one answered from the ledger costs none.
2163
+ work,
2164
+ // The call's copy answers, shared by every recipient of a locale group.
2165
+ copies,
2166
+ // Resolves the call's locale groups once, when the first recipient needs a send.
2167
+ ensureCopies) => {
2168
+ const request = sendRequestOf(raw);
2169
+ const fingerprint = notificationSendFingerprint(request);
2170
+ const key = request.idempotencyKey;
2171
+ for (let attempt = 0; attempt < maxLedgerAttempts; attempt += 1) {
2172
+ const at = nowMs();
2173
+ const held = await lookup(key, at);
2174
+ if (held !== undefined) {
2175
+ if (held.send.fingerprint !== fingerprint) {
2176
+ throw new NotificationError('NOTIFICATIONS_IDEMPOTENCY_CONFLICT', {
2177
+ message: `Key ${key} is stored for another request.`,
2178
+ });
2179
+ }
2180
+ const { send } = held;
2181
+ if (send.rows.length === 0) {
2182
+ if (send.claim !== undefined && Date.parse(send.claim.leaseUntil) > at) {
2183
+ return { done: { outcome: { status: 'in-flight' } } };
2184
+ }
2185
+ await store.updateSend({ key, version: held.version, send: undefined });
2186
+ continue;
2187
+ }
2188
+ const wasDelivered = notificationSendOutcome(send)?.status === 'delivered';
2189
+ // A platform row whose lease ended (a call that stopped mid-way) is
2190
+ // this call's to replay, as a retried `deliver` replays it.
2191
+ const expired = send.rows.filter((row) => eligible('retry', row, at));
2192
+ if (expired.length > 0) {
2193
+ if (work.left <= 0) {
2194
+ return { done: { notAttempted: 'count-budget' } };
2195
+ }
2196
+ // Each delivery row is an attempt: a player's devices each cost one.
2197
+ work.left -= expired.length;
2198
+ for (const row of expired) {
2199
+ if (!await pace.take(row.provider)) {
2200
+ return { done: { notAttempted: 'provider-rate' } };
2201
+ }
2202
+ }
2203
+ return { key, rows: expired, mode: 'retry', refusals: [], wasDelivered };
2204
+ }
2205
+ const stored = notificationSendOutcome(send) ?? noEndpoint();
2206
+ return {
2207
+ done: {
2208
+ outcome: stored.status === 'delivered'
2209
+ ? { status: 'delivered', alreadyProcessed: true }
2210
+ : stored,
2211
+ },
2212
+ };
2213
+ }
2214
+ // Spent out of the call's budget only once the gate allowed a row.
2215
+ if (work.left <= 0) {
2216
+ return { done: { notAttempted: 'count-budget' } };
2217
+ }
2218
+ // The copy is resolved only once a recipient needs a send, so a retry
2219
+ // whose recipients were all sent never asks the resolver; a resolver
2220
+ // that fails fails this recipient, retryable, before its key is made.
2221
+ await ensureCopies();
2222
+ const send = {
2223
+ key,
2224
+ fingerprint,
2225
+ request,
2226
+ origin: { kind: 'send' },
2227
+ scheduled: false,
2228
+ rows: [],
2229
+ claim: { token: newToken(), leaseUntil: formatNotificationInstant(at + leaseMs) },
2230
+ changedAt: formatNotificationInstant(at),
2231
+ };
2232
+ const version = await store.createSend(send);
2233
+ if (version === undefined) {
2234
+ // A concurrent send created the key first: answer from its state.
2235
+ continue;
2236
+ }
2237
+ let planned;
2238
+ try {
2239
+ planned = await plan(send.request, send.origin, at, undefined, copies);
2240
+ }
2241
+ catch (error) {
2242
+ // Nothing was sent: the key is released so a retry evaluates again.
2243
+ await releaseQuietly(key, version);
2244
+ if (error instanceof NotificationSendInputError) {
2245
+ throw invalidInput(error.message, error);
2246
+ }
2247
+ throw error;
2248
+ }
2249
+ if (planned.rows.length === 0) {
2250
+ await store.updateSend({ key, version, send: undefined });
2251
+ return { done: { outcome: bestNotificationOutcome(planned.refusals) ?? noEndpoint() } };
2252
+ }
2253
+ // The call's and the provider's app-wide budgets are spent once the gate
2254
+ // allowed a delivery, and before any row is stored: a recipient over
2255
+ // either leaves nothing behind, and the same call retried sends it once
2256
+ // the budget frees. A refused recipient spends neither.
2257
+ const allowed = planned.rows.filter((row) => row.state.status === 'in-flight');
2258
+ if (allowed.length > 0) {
2259
+ // Taken at once, so recipients planned side by side cannot all pass it.
2260
+ let over;
2261
+ if (work.left <= 0) {
2262
+ over = 'count-budget';
2263
+ }
2264
+ else {
2265
+ work.left -= 1;
2266
+ let paced;
2267
+ try {
2268
+ paced = await pace.take(provider);
2269
+ }
2270
+ catch (error) {
2271
+ // The rate could not be read (the store failed): nothing was
2272
+ // sent, so the attempt goes back and the key is released.
2273
+ work.left += 1;
2274
+ await releaseQuietly(key, version);
2275
+ throw error;
2276
+ }
2277
+ if (!paced) {
2278
+ work.left += 1;
2279
+ over = 'provider-rate';
2280
+ }
2281
+ }
2282
+ if (over !== undefined) {
2283
+ await releaseQuietly(key, version);
2284
+ return { done: { notAttempted: over } };
2285
+ }
2286
+ }
2287
+ const { claim: _claim, ...unclaimed } = send;
2288
+ const stored = {
2289
+ ...unclaimed,
2290
+ rows: planned.rows,
2291
+ changedAt: formatNotificationInstant(at),
2292
+ };
2293
+ if (!await store.updateSend({ key, version, send: stored })) {
2294
+ // Another caller took the key over: this call delivers nothing for it,
2295
+ // so the attempt and the provider's rate it reserved go back.
2296
+ if (allowed.length > 0) {
2297
+ work.left += 1;
2298
+ pace.refund(provider);
2299
+ }
2300
+ return { done: { outcome: { status: 'in-flight' } } };
2301
+ }
2302
+ // The first allowed row was counted above; each further device is one
2303
+ // more attempt and one more of the provider's rate. A device over
2304
+ // either stays queued: the scheduler sends it at its own pace.
2305
+ const now = allowed.slice(0, 1);
2306
+ for (const row of allowed.slice(1)) {
2307
+ if (work.left <= 0) {
2308
+ break;
2309
+ }
2310
+ work.left -= 1;
2311
+ let paced = false;
2312
+ try {
2313
+ paced = await pace.take(row.provider);
2314
+ }
2315
+ catch (error) {
2316
+ // The device's row is stored: it stays queued for the scheduler.
2317
+ report(error, 'deliverBulk.pace', {
2318
+ sendKey: key,
2319
+ provider: row.provider,
2320
+ rowKey: row.key,
2321
+ });
2322
+ }
2323
+ if (!paced) {
2324
+ work.left += 1;
2325
+ break;
2326
+ }
2327
+ now.push(row);
2328
+ }
2329
+ return {
2330
+ key,
2331
+ rows: now,
2332
+ mode: 'job',
2333
+ refusals: planned.refusals,
2334
+ wasDelivered: false,
2335
+ };
2336
+ }
2337
+ throw busy(key);
2338
+ };
2339
+ /**
2340
+ * The flush of one batch: the requests in the provider's call groups
2341
+ * (`bulk.callKey`, one title and body a call on Devvit), one `sendBatch`
2342
+ * per group and `maxRecipients`, or one `send` per request below
2343
+ * `minRecipients` and inside the provider's hold window. The gate runs
2344
+ * again for each request as its call starts, and no call starts past the
2345
+ * claims' deadline: a request the gate now refuses, or one past the
2346
+ * deadline, answers `not-sent` (it is replayed shortly). Every answer is one of the five kinds; a
2347
+ * missing or broken one, or a call that threw, is `ambiguous`.
2348
+ */
2349
+ const flushBulk = async (provider, requests, calls,
2350
+ // Milliseconds from now in which the last provider call may start, so every claim outlives the calls.
2351
+ startWithinMs,
2352
+ // Each request as the gate allows it at a later instant (undefined: no longer).
2353
+ regates = []) => {
2354
+ const capability = provider.delivery.bulk;
2355
+ const startedAt = nowMs();
2356
+ const sendBatch = provider.delivery.sendBatch;
2357
+ const answers = requests.map(() => ({
2358
+ status: 'ambiguous',
2359
+ }));
2360
+ const notSent = { status: 'not-sent' };
2361
+ /** The request as the gate allows it at `at`, or `undefined`. */
2362
+ const current = (index, at) => {
2363
+ const regate = regates[index];
2364
+ try {
2365
+ return regate === undefined ? requests[index] : regate(at);
2366
+ }
2367
+ catch {
2368
+ return undefined;
2369
+ }
2370
+ };
2371
+ const singles = async (indexes) => {
2372
+ const limiter = createNotificationLimiter(bulkConcurrency);
2373
+ await Promise.all(indexes.map(async (index) => {
2374
+ await limiter.acquire();
2375
+ try {
2376
+ const at = nowMs();
2377
+ const request = at - startedAt >= startWithinMs ? undefined : current(index, at);
2378
+ if (request === undefined) {
2379
+ answers[index] = notSent;
2380
+ return;
2381
+ }
2382
+ calls.single += 1;
2383
+ answers[index] = await sendOne(provider, request);
2384
+ }
2385
+ finally {
2386
+ limiter.release();
2387
+ }
2388
+ }));
2389
+ };
2390
+ if (capability === undefined
2391
+ || sendBatch === undefined
2392
+ || requests.length < (capability.minRecipients ?? 2)
2393
+ || isInsideNotificationHoldWindow(capability.holdWindow, nowMs())) {
2394
+ // Fewer than the minimum after the gate, or the hold window opened while the batch gathered.
2395
+ await singles(requests.map((_, index) => index));
2396
+ return answers;
2397
+ }
2398
+ const groups = new Map();
2399
+ requests.forEach((request, index) => {
2400
+ let key = '';
2401
+ try {
2402
+ key = capability.callKey?.(request) ?? '';
2403
+ }
2404
+ catch {
2405
+ key = '';
2406
+ }
2407
+ const group = groups.get(key) ?? [];
2408
+ group.push(index);
2409
+ groups.set(key, group);
2410
+ });
2411
+ for (const group of groups.values()) {
2412
+ for (let start = 0; start < group.length; start += capability.maxRecipients) {
2413
+ const at = nowMs();
2414
+ const left = startWithinMs - (at - startedAt);
2415
+ const sending = [];
2416
+ for (const index of group.slice(start, start + capability.maxRecipients)) {
2417
+ const request = left > 0 ? current(index, at) : undefined;
2418
+ if (request === undefined) {
2419
+ answers[index] = notSent;
2420
+ }
2421
+ else {
2422
+ sending.push({ index, request });
2423
+ }
2424
+ }
2425
+ if (sending.length === 0) {
2426
+ continue;
2427
+ }
2428
+ if (sending.length < (capability.minRecipients ?? 2)) {
2429
+ await singles(sending.map((entry) => entry.index));
2430
+ continue;
2431
+ }
2432
+ const slice = sending.map((entry) => entry.request);
2433
+ calls.batch += 1;
2434
+ let raw;
2435
+ try {
2436
+ raw = await sendBatch.call(provider.delivery, slice, { startWithinMs: left });
2437
+ }
2438
+ catch (error) {
2439
+ // The provider may have sent any of them: one rule for every failure it cannot prove.
2440
+ report(error, 'provider.sendBatch', { sendKey: slice[0]?.sendKey ?? '', provider: provider.entry.provider }, true);
2441
+ continue;
2442
+ }
2443
+ const list = Array.isArray(raw) ? raw : [];
2444
+ if (list.length !== slice.length) {
2445
+ report(new TypeError(`The provider answered ${list.length} answers for a batch of ${slice.length}.`), 'provider.sendBatch', { sendKey: slice[0]?.sendKey ?? '', provider: provider.entry.provider });
2446
+ continue;
2447
+ }
2448
+ sending.forEach(({ index, request }, position) => {
2449
+ const known = deliveryAnswerOf(list[position]);
2450
+ if (known === undefined) {
2451
+ report(new TypeError('The provider answered no known delivery answer.'), 'provider.sendBatch', {
2452
+ sendKey: request.sendKey,
2453
+ provider: provider.entry.provider,
2454
+ rowKey: request.idempotencyKey,
2455
+ });
2456
+ }
2457
+ answers[index] = known ?? { status: 'ambiguous' };
2458
+ });
2459
+ }
2460
+ }
2461
+ return answers;
2462
+ };
2463
+ /**
2464
+ * Runs the prepared recipients of one batch: each row through `runRow`
2465
+ * (claim, gate, provider call, settle), with their provider calls
2466
+ * gathered into `flushBulk` for a provider with a bulk capability. A
2467
+ * send's rows run one after the other, in rounds, so two rows of one send
2468
+ * never contend for its key. Answers each recipient as `deliver` would.
2469
+ */
2470
+ const runBulkBatch = async (provider, prepared, calls, copies) => {
2471
+ const limiter = createNotificationLimiter(bulkConcurrency);
2472
+ const capability = provider.delivery.bulk;
2473
+ const state = prepared.map((entry) => ({
2474
+ refusals: [...entry.refusals],
2475
+ errors: [],
2476
+ callerError: undefined,
2477
+ settleFailed: false,
2478
+ }));
2479
+ const rounds = Math.max(0, ...prepared.map((entry) => entry.rows.length));
2480
+ for (let round = 0; round < rounds; round += 1) {
2481
+ const members = prepared
2482
+ .map((entry, index) => ({ entry, index, row: entry.rows[round] }))
2483
+ .filter((member) => member.row !== undefined);
2484
+ // Only a provider with a bulk path gathers calls, for a round of at
2485
+ // least its minimum and outside its hold window. Any other round sends
2486
+ // each recipient as its gate allows it and settles it at once, so no
2487
+ // claim waits on a queue of single calls past its lease.
2488
+ const gathers = capability !== undefined
2489
+ && provider.delivery.sendBatch !== undefined
2490
+ && members.length >= (capability.minRecipients ?? 2)
2491
+ && !isInsideNotificationHoldWindow(capability.holdWindow, nowMs());
2492
+ const barrier = createNotificationBatchBarrier(members.length, async (items) => {
2493
+ // A recipient whose claim has less than half a lease left (it may run
2494
+ // out before the call answers, and another holder may then own its
2495
+ // row) is not sent. Nothing was sent, so it waits a moment and is
2496
+ // replayed (its own claim settles only while it still holds the row).
2497
+ // The gate runs again for each recipient at the batch's own time
2498
+ // (quiet hours or the night window may have begun while it gathered),
2499
+ // with the inputs its first check read. A recipient it no longer
2500
+ // allows is not sent either: it is replayed, and its replay's gate
2501
+ // defers or refuses it.
2502
+ const now = nowMs();
2503
+ const checked = items.map((item) => {
2504
+ if (item.claimedUntil - now < leaseMs / 2) {
2505
+ return { item, reason: 'has less than half a claim lease left' };
2506
+ }
2507
+ let request;
2508
+ try {
2509
+ request = item.regate(now);
2510
+ }
2511
+ catch {
2512
+ request = undefined;
2513
+ }
2514
+ return request === undefined
2515
+ ? { item, reason: 'is no longer allowed by the gate, or its gate inputs are too old' }
2516
+ : { item, request };
2517
+ });
2518
+ const fresh = checked.flatMap((entry) => ('request' in entry ? [entry] : []));
2519
+ // The last call starts while every fresh claim still has a quarter lease left.
2520
+ const startWithinMs = Math.min(...fresh.map((entry) => entry.item.claimedUntil)) - leaseMs / 4 - now;
2521
+ const answers = await flushBulk(provider, fresh.map((entry) => entry.request), calls, startWithinMs, fresh.map((entry) => entry.item.regate));
2522
+ let next = 0;
2523
+ return checked.map((entry) => {
2524
+ if ('request' in entry) {
2525
+ next += 1;
2526
+ return answers[next - 1] ?? { status: 'ambiguous' };
2527
+ }
2528
+ report(new Error(`A bulk recipient ${entry.reason} when its batch is sent; it is not sent now.`), 'dispatch.stale', {
2529
+ sendKey: entry.item.request.sendKey,
2530
+ provider: provider.entry.provider,
2531
+ rowKey: entry.item.request.idempotencyKey,
2532
+ });
2533
+ return { status: 'not-sent' };
2534
+ });
2535
+ });
2536
+ await Promise.all(members.map(async ({ entry, index, row }) => {
2537
+ const slot = state[index];
2538
+ if (slot === undefined) {
2539
+ return;
2540
+ }
2541
+ await limiter.acquire();
2542
+ let holding = true;
2543
+ let arrived = false;
2544
+ const release = () => {
2545
+ if (holding) {
2546
+ holding = false;
2547
+ limiter.release();
2548
+ }
2549
+ };
2550
+ try {
2551
+ const { refusal } = await runRow(entry.key, row.provider, row.key, entry.mode, copies, async (recipient, request, topic, decision, at, claimedUntil, regate) => {
2552
+ arrived = true;
2553
+ if (!gathers) {
2554
+ calls.single += 1;
2555
+ return callProvider(recipient, request, topic, decision, at);
2556
+ }
2557
+ // The slot goes back while the batch gathers, so the barrier never waits on a recipient that cannot run.
2558
+ release();
2559
+ const answer = await barrier.arrive({
2560
+ request: deliveryRequestOf(recipient, request, topic, decision, at),
2561
+ claimedUntil: claimedUntil ?? at,
2562
+ regate: (when) => {
2563
+ // The consent, history, and host permission were read at `at`: past a short age
2564
+ // they may be stale, so the recipient is replayed with fresh reads instead.
2565
+ if (when - at > bulkGateFreshMs) {
2566
+ return undefined;
2567
+ }
2568
+ const again = regate?.(when);
2569
+ return again?.status === 'allowed'
2570
+ ? deliveryRequestOf(recipient, request, topic, again, when)
2571
+ : undefined;
2572
+ },
2573
+ });
2574
+ await limiter.acquire();
2575
+ holding = true;
2576
+ return answer;
2577
+ });
2578
+ if (refusal !== undefined) {
2579
+ slot.refusals.push(refusal);
2580
+ }
2581
+ }
2582
+ catch (error) {
2583
+ if (error instanceof NotificationError && error.code === 'NOTIFICATIONS_INVALID_INPUT') {
2584
+ slot.callerError ??= error;
2585
+ }
2586
+ else {
2587
+ slot.errors.push({ error, rowKey: row.key, provider: row.provider });
2588
+ slot.settleFailed ||= error instanceof NotificationSettleError;
2589
+ }
2590
+ }
2591
+ finally {
2592
+ if (!arrived) {
2593
+ barrier.leave();
2594
+ }
2595
+ release();
2596
+ }
2597
+ }));
2598
+ }
2599
+ return Promise.all(prepared.map(async (entry, index) => {
2600
+ const slot = state[index];
2601
+ // Every row failed before its provider call: the first failure is the
2602
+ // recipient's. A row whose answer could not be stored may have been
2603
+ // delivered, so it answers from the ledger, as `deliver` does. Every
2604
+ // other failure is reported here, as `runInline` reports it.
2605
+ const errors = slot?.errors ?? [];
2606
+ const thrown = slot?.callerError
2607
+ ?? (errors.length > 0 && errors.length === entry.rows.length && slot?.settleFailed !== true
2608
+ ? errors[0]?.error
2609
+ : undefined);
2610
+ for (const failure of errors) {
2611
+ if (failure.error !== thrown) {
2612
+ report(failure.error, 'dispatch', { sendKey: entry.key, provider: failure.provider, rowKey: failure.rowKey });
2613
+ }
2614
+ }
2615
+ if (thrown !== undefined) {
2616
+ return { error: thrown };
2617
+ }
2618
+ let settled;
2619
+ try {
2620
+ settled = await settle(entry.key, slot?.refusals ?? []);
2621
+ }
2622
+ catch (error) {
2623
+ // The read failed, not the send: this recipient alone answers it.
2624
+ return { error };
2625
+ }
2626
+ return {
2627
+ outcome: entry.wasDelivered && settled.status === 'delivered'
2628
+ ? { status: 'delivered', alreadyProcessed: true }
2629
+ : settled,
2630
+ };
2631
+ }));
2632
+ };
2633
+ const deliverBulk = async (raw, options = {}) => {
2634
+ const { input: bulk, keys } = parseNotificationBulkRequest(raw);
2635
+ refuseNoticeTopic(bulk.topic);
2636
+ const provider = registry.provider(bulk.provider);
2637
+ if (provider === undefined) {
2638
+ throw invalidInput(`This backend has no notifications provider ${bulk.provider}.`);
2639
+ }
2640
+ const capability = provider.delivery.bulk;
2641
+ const batchSize = Math.min(bulk.batchSize ?? capability?.maxRecipients ?? defaultBulkBatchSize, capability?.maxRecipients ?? maxNotificationRowsPerBulkBatch);
2642
+ const budgetMs = typeof options.budgetMs === 'number' && Number.isFinite(options.budgetMs) && options.budgetMs > 0
2643
+ ? options.budgetMs
2644
+ : defaultSchedulerBudgetMs;
2645
+ const work = {
2646
+ left: typeof options.maxRecipients === 'number' && Number.isFinite(options.maxRecipients)
2647
+ ? Math.max(0, Math.floor(options.maxRecipients))
2648
+ : Number.POSITIVE_INFINITY,
2649
+ };
2650
+ const lookups = {
2651
+ left: typeof options.maxLookups === 'number' && Number.isFinite(options.maxLookups)
2652
+ ? Math.max(0, Math.floor(options.maxLookups))
2653
+ : Number.POSITIVE_INFINITY,
2654
+ };
2655
+ const elapsed = input.elapsedMs ?? (() => globalThis.performance.now());
2656
+ const startedAt = elapsed();
2657
+ // Recipients are prepared concurrently: their takes are made one at a
2658
+ // time, so the budget is read once and never spent twice.
2659
+ const budgets = paceBudgets();
2660
+ let paced = Promise.resolve();
2661
+ const pace = {
2662
+ take(provider) {
2663
+ const taken = paced.then(() => budgets.take(provider));
2664
+ paced = taken.catch(() => undefined);
2665
+ return taken;
2666
+ },
2667
+ refund: (provider) => budgets.refund(provider),
2668
+ };
2669
+ const calls = { batch: 0, single: 0 };
2670
+ // One copy answer per locale group for the whole call, shared by
2671
+ // planning, the provider calls, and the flush-time gate checks. It is
2672
+ // resolved once, when the first recipient needs a send.
2673
+ const copies = new Map();
2674
+ const lookupLimit = lookups.left;
2675
+ let primed;
2676
+ const ensureCopies = () => primed ??= primeBulkCopies(provider, {
2677
+ topic: bulk.topic,
2678
+ idempotencyKey: bulk.idempotencyKey,
2679
+ ...(bulk.templateData === undefined ? {} : { templateData: bulk.templateData }),
2680
+ },
2681
+ // Only well-formed locales of recipients within the read budget: a
2682
+ // malformed one fails its own recipient's request check, never the
2683
+ // whole call through the resolver.
2684
+ bulk.recipients
2685
+ .slice(0, Number.isFinite(lookupLimit) ? lookupLimit : undefined)
2686
+ .filter((recipient) => recipient.templateData === undefined)
2687
+ .map((recipient) => recipient.locale ?? bulk.locale)
2688
+ .filter((locale) => isBoundedText(locale)), copies);
2689
+ const answers = keys.map(() => undefined);
2690
+ const shared = {
2691
+ ...(bulk.templateData === undefined ? {} : { templateData: bulk.templateData }),
2692
+ ...(bulk.data === undefined ? {} : { data: bulk.data }),
2693
+ ...(bulk.locale === undefined ? {} : { locale: bulk.locale }),
2694
+ ...(bulk.link === undefined ? {} : { link: bulk.link }),
2695
+ };
2696
+ for (let start = 0; start < keys.length; start += batchSize) {
2697
+ const end = Math.min(start + batchSize, keys.length);
2698
+ if (start > 0 && elapsed() - startedAt >= budgetMs) {
2699
+ for (let index = start; index < keys.length; index += 1) {
2700
+ answers[index] = { notAttempted: 'time-budget' };
2701
+ }
2702
+ break;
2703
+ }
2704
+ const ready = [];
2705
+ const limiter = createNotificationLimiter(bulkConcurrency);
2706
+ await Promise.all(bulk.recipients.slice(start, end).map(async (recipient, offset) => {
2707
+ const index = start + offset;
2708
+ // Every recipient's key is read once; past the read budget, none is.
2709
+ if (lookups.left <= 0) {
2710
+ answers[index] = { notAttempted: 'count-budget' };
2711
+ return;
2712
+ }
2713
+ lookups.left -= 1;
2714
+ await limiter.acquire();
2715
+ try {
2716
+ const made = await prepareBulkRecipient({
2717
+ ...shared,
2718
+ ...recipient,
2719
+ idempotencyKey: keys[index] ?? '',
2720
+ topic: bulk.topic,
2721
+ provider: bulk.provider,
2722
+ }, bulk.provider, pace, work, copies, ensureCopies);
2723
+ if ('done' in made) {
2724
+ answers[index] = made.done;
2725
+ }
2726
+ else if (made.rows.length === 0) {
2727
+ answers[index] = { outcome: await settle(made.key, [...made.refusals]) };
2728
+ }
2729
+ else {
2730
+ ready.push({ index, prepared: made });
2731
+ }
2732
+ }
2733
+ catch (error) {
2734
+ answers[index] = { error };
2735
+ }
2736
+ finally {
2737
+ limiter.release();
2738
+ }
2739
+ }));
2740
+ ready.sort((left, right) => left.index - right.index);
2741
+ const ran = await runBulkBatch(provider, ready.map((entry) => entry.prepared), calls, copies);
2742
+ ready.forEach((entry, position) => {
2743
+ answers[entry.index] = ran[position] ?? { error: new Error('The batch answered no result.') };
2744
+ });
2745
+ }
2746
+ const recipients = bulk.recipients.map((recipient, index) => {
2747
+ const base = { player: recipient.player, idempotencyKey: keys[index] ?? '' };
2748
+ const answer = answers[index] ?? { notAttempted: 'time-budget' };
2749
+ if ('outcome' in answer) {
2750
+ return { ...base, status: 'answered', outcome: answer.outcome };
2751
+ }
2752
+ if ('notAttempted' in answer) {
2753
+ return { ...base, status: 'not-attempted', reason: answer.notAttempted };
2754
+ }
2755
+ const { error } = answer;
2756
+ if (!(error instanceof NotificationError)) {
2757
+ report(error, 'deliverBulk', base.idempotencyKey);
2758
+ }
2759
+ return {
2760
+ ...base,
2761
+ status: 'failed',
2762
+ error: error instanceof NotificationError
2763
+ ? { code: error.code, message: error.message, retryable: error.retryable }
2764
+ : {
2765
+ code: 'NOTIFICATIONS_TEMPORARILY_UNAVAILABLE',
2766
+ message: 'The send failed before an answer could be stored; retry the bulk send.',
2767
+ retryable: true,
2768
+ },
2769
+ };
2770
+ });
2771
+ return { recipients, summary: bulkSummaryOf(recipients), providerCalls: calls };
2772
+ };
1889
2773
  return {
1890
2774
  providers: () => registry.snapshot(),
1891
2775
  async recordConsent(session, record) {
@@ -2130,6 +3014,7 @@ export function createNotificationService(input) {
2130
3014
  }
2131
3015
  throw busy(key);
2132
3016
  },
3017
+ deliverBulk,
2133
3018
  async dispatch(job) {
2134
3019
  if (!isPlainObject(job)
2135
3020
  || !isBoundedText(job.sendKey)
@@ -2281,7 +3166,7 @@ export function createNotificationService(input) {
2281
3166
  }
2282
3167
  // A placeholder the fire would throw on is refused now, before the
2283
3168
  // previous reminder's send is stopped or this one is written.
2284
- assertReminderCopyFills(declaring, reminder.topic, reminder.templateData, catalog);
3169
+ assertReminderCopyFills(declaring, reminder.topic, reminder.templateData, catalog, !customCopy);
2285
3170
  // One reminder per key: setting it replaces the previous one and, before
2286
3171
  // the write, stops the previous one's send that waits; a scheduler run
2287
3172
  // that fires the previous one meanwhile then fails to mark it and stops
@@ -2369,14 +3254,17 @@ export function createNotificationService(input) {
2369
3254
  const touched = new Map();
2370
3255
  // One pace for the whole run: rows taken in an earlier round count.
2371
3256
  const pace = paceBudgets();
3257
+ // One copy cache for the whole run: a reminder's first check and its
3258
+ // dispatch, and recipients that share a locale, share a resolver call.
3259
+ const copies = new Map();
2372
3260
  let failed = 0;
2373
3261
  for (;;) {
2374
3262
  let more;
2375
3263
  let progressed;
2376
3264
  try {
2377
- const entries = await runScheduleEntries(limit, touched);
3265
+ const entries = await runScheduleEntries(limit, touched, copies);
2378
3266
  failed += entries.failed;
2379
- const dispatched = await dispatchDueRows({ limit }, pace);
3267
+ const dispatched = await dispatchDueRows({ limit }, pace, copies);
2380
3268
  failed += dispatched.failed;
2381
3269
  for (const result of dispatched.results) {
2382
3270
  touched.set(result.sendKey, result.outcome);
@@ -2425,7 +3313,9 @@ export function createNotificationService(input) {
2425
3313
  * is `NOTIFICATIONS_INVALID_INPUT` now, not a reminder that is stored and
2426
3314
  * then dropped when it fires.
2427
3315
  */
2428
- function assertReminderCopyFills(providers, topicId, templateData, catalog) {
3316
+ function assertReminderCopyFills(providers, topicId, templateData, catalog,
3317
+ /** False under a project's copy resolver, whose text replaces the catalog's copy. */
3318
+ catalogCopy) {
2429
3319
  const data = templateData ?? {};
2430
3320
  for (const provider of providers) {
2431
3321
  const hostTemplate = provider.entry.content === 'host-template';
@@ -2442,7 +3332,7 @@ function assertReminderCopyFills(providers, topicId, templateData, catalog) {
2442
3332
  }
2443
3333
  const texts = mapped
2444
3334
  ? [mappedTitle, mappedBody]
2445
- : topic.copy === undefined
3335
+ : topic.copy === undefined || !catalogCopy
2446
3336
  ? []
2447
3337
  : Object.values(topic.copy).flatMap((localeCopy) => [localeCopy.title, localeCopy.body]);
2448
3338
  for (const text of texts) {
@@ -2760,6 +3650,14 @@ function backedOff(row, at) {
2760
3650
  * permission-order test in service.test.ts covers both sides).
2761
3651
  */
2762
3652
  const earlyGateReasons = new Set(['topic-unknown', 'channel-unsupported', 'purpose-unsupported', 'no-endpoint', 'link-missing']);
3653
+ /**
3654
+ * The copy the gate renders while it decides whether a recipient is allowed
3655
+ * at all; the resolver's answer replaces it before anything is sent, so the
3656
+ * copy resolver is asked only for recipients the gate would allow.
3657
+ */
3658
+ const copyStandIn = Object.freeze({
3659
+ copy: Object.freeze({ title: 'copy', body: 'copy' }),
3660
+ });
2763
3661
  /** Writes that must not fail a settled delivery are tried this often. */
2764
3662
  const maxSideWriteAttempts = 3;
2765
3663
  /** Runs `write` up to `maxSideWriteAttempts` times with a short pause; reports only the last failure and never throws. */
@@ -2796,6 +3694,8 @@ function deliveryAnswerOf(answer) {
2796
3694
  : { status: 'accepted' };
2797
3695
  case 'unregistered':
2798
3696
  return { status: 'unregistered' };
3697
+ case 'not-sent':
3698
+ return { status: 'not-sent' };
2799
3699
  case 'rate-limited':
2800
3700
  return typeof answer.retryAfterSeconds === 'number'
2801
3701
  ? { status: 'rate-limited', retryAfterSeconds: answer.retryAfterSeconds }
@@ -2812,10 +3712,68 @@ function deliveryAnswerOf(answer) {
2812
3712
  }
2813
3713
  : { status: 'rejected', reason: answer.reason };
2814
3714
  case 'ambiguous':
2815
- return typeof answer.providerCode === 'string'
2816
- ? { status: 'ambiguous', providerCode: answer.providerCode.slice(0, 64) }
2817
- : { status: 'ambiguous' };
3715
+ return {
3716
+ status: 'ambiguous',
3717
+ ...(typeof answer.providerCode === 'string' ? { providerCode: answer.providerCode.slice(0, 64) } : {}),
3718
+ ...(answer.unattributed === true ? { unattributed: true } : {}),
3719
+ };
2818
3720
  default:
2819
3721
  return undefined;
2820
3722
  }
2821
3723
  }
3724
+ /** Recipients prepared and run at once in a bulk batch (each one's ledger reads and writes). */
3725
+ const bulkConcurrency = 8;
3726
+ /**
3727
+ * How old a bulk recipient's gate inputs (consent, history, host permission)
3728
+ * may be when its call starts; an older one is replayed with fresh reads.
3729
+ */
3730
+ const bulkGateFreshMs = 30_000;
3731
+ /** Recipients per batch for a provider that sends one by one. */
3732
+ const defaultBulkBatchSize = 100;
3733
+ /** The most recipients one bulk batch runs for a provider that sends one by one. */
3734
+ const maxNotificationRowsPerBulkBatch = 1_000;
3735
+ function bulkSummaryOf(recipients) {
3736
+ const summary = {
3737
+ delivered: 0,
3738
+ alreadyProcessed: 0,
3739
+ deferred: 0,
3740
+ inFlight: 0,
3741
+ suppressed: 0,
3742
+ unavailable: 0,
3743
+ notAttempted: 0,
3744
+ failed: 0,
3745
+ };
3746
+ for (const recipient of recipients) {
3747
+ if (recipient.status === 'not-attempted') {
3748
+ summary.notAttempted += 1;
3749
+ }
3750
+ else if (recipient.status === 'failed') {
3751
+ summary.failed += 1;
3752
+ }
3753
+ else {
3754
+ switch (recipient.outcome.status) {
3755
+ case 'delivered':
3756
+ if (recipient.outcome.alreadyProcessed) {
3757
+ summary.alreadyProcessed += 1;
3758
+ }
3759
+ else {
3760
+ summary.delivered += 1;
3761
+ }
3762
+ break;
3763
+ case 'deferred':
3764
+ summary.deferred += 1;
3765
+ break;
3766
+ case 'in-flight':
3767
+ summary.inFlight += 1;
3768
+ break;
3769
+ case 'suppressed':
3770
+ summary.suppressed += 1;
3771
+ break;
3772
+ case 'unavailable':
3773
+ summary.unavailable += 1;
3774
+ break;
3775
+ }
3776
+ }
3777
+ }
3778
+ return summary;
3779
+ }