@ultimat3/notify 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +29 -5
- package/README.md +94 -2
- package/package.json +18 -5
- package/src/apns.ts +145 -0
- package/src/attempt.ts +39 -7
- package/src/channel-push.ts +44 -5
- package/src/errors.ts +48 -1
- package/src/fanout-digest.ts +15 -11
- package/src/fanout.ts +2 -2
- package/src/fcm.ts +211 -0
- package/src/index.ts +36 -1
- package/src/ledger-pg.ts +55 -16
- package/src/ledger.ts +59 -19
- package/src/native-jwt.ts +63 -0
- package/src/native-push.ts +173 -0
- package/src/native-transport.ts +99 -0
- package/src/notifier.ts +2 -1
- package/src/plan.ts +2 -0
- package/src/push-device-pg.ts +126 -0
- package/src/push-device.ts +125 -0
- package/src/push-errors.ts +83 -0
- package/src/push-message.ts +109 -0
- package/src/push-register.ts +167 -0
- package/src/token-cache.ts +57 -0
package/src/fanout-digest.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// Exactly one run opens a given window, so exactly one sleeps on it — which is what stops fifty
|
|
5
5
|
// events in one hour from scheduling fifty flushes of the same digest.
|
|
6
6
|
|
|
7
|
+
import type { StepOutput } from '@ultimat3/jobs';
|
|
7
8
|
import { attemptDelivery } from './attempt';
|
|
8
9
|
import type { NotifyChannel } from './channel';
|
|
9
10
|
import type { DigestSlot } from './digest';
|
|
@@ -78,7 +79,7 @@ export async function flushDigest<Params>(input: DigestFlush<Params>): Promise<v
|
|
|
78
79
|
digest.drain(slotFor(id), endsAt),
|
|
79
80
|
);
|
|
80
81
|
if (batch.length === 0) continue;
|
|
81
|
-
const events = rehydrate
|
|
82
|
+
const events = rehydrate(batch, plan.parseParams);
|
|
82
83
|
const newest = events[events.length - 1] ?? event;
|
|
83
84
|
const sent = await step.run(`digest-send:${channel.name}:${id}`, (signal) =>
|
|
84
85
|
attemptDelivery(
|
|
@@ -86,10 +87,11 @@ export async function flushDigest<Params>(input: DigestFlush<Params>): Promise<v
|
|
|
86
87
|
ledger: notifyStores().ledger,
|
|
87
88
|
// The window, not the event: a digest is one delivery for many events, so its ledger
|
|
88
89
|
// identity is the slot plus the window it closed. Two different windows for the same
|
|
89
|
-
// recipient are two deliveries and must not dedupe into one.
|
|
90
|
+
// recipient are two deliveries and must not dedupe into one. THIS recipient's end, never
|
|
91
|
+
// the run's maximum: two runs whose maxima coincide would share one key across windows.
|
|
90
92
|
claim: {
|
|
91
93
|
notifier: plan.name,
|
|
92
|
-
key: `digest:${group}:${String(
|
|
94
|
+
key: `digest:${group}:${String(endsAt)}`,
|
|
93
95
|
recipient: id,
|
|
94
96
|
channel: channel.name,
|
|
95
97
|
},
|
|
@@ -109,14 +111,16 @@ export async function flushDigest<Params>(input: DigestFlush<Params>): Promise<v
|
|
|
109
111
|
}
|
|
110
112
|
|
|
111
113
|
/**
|
|
112
|
-
* A step
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
* The cast is the digest store's untyped edge: it holds `NotifyEvent<unknown>` because one store
|
|
117
|
-
* serves every notifier, and what comes out of a slot is exactly what this notifier put in.
|
|
114
|
+
* A step answers the stored form, so the drained batch holds `at` and any date in `params` as ISO
|
|
115
|
+
* strings on every attempt, and a channel is promised a `Date` and `Params`: both rebuilt here,
|
|
116
|
+
* `params` through the notifier's own `input`, the parse a job's payload gets off the queue.
|
|
118
117
|
*/
|
|
119
118
|
const rehydrate = <Params>(
|
|
120
|
-
batch: readonly NotifyEvent<unknown>[]
|
|
119
|
+
batch: StepOutput<readonly NotifyEvent<unknown>[]>,
|
|
120
|
+
parseParams: (raw: unknown) => Params,
|
|
121
121
|
): readonly NotifyEvent<Params>[] =>
|
|
122
|
-
batch.map((entry) => ({
|
|
122
|
+
batch.map((entry) => ({
|
|
123
|
+
...entry,
|
|
124
|
+
at: new Date(entry.at),
|
|
125
|
+
params: parseParams(entry.params),
|
|
126
|
+
}));
|
package/src/fanout.ts
CHANGED
|
@@ -25,8 +25,8 @@ export async function runFanout<Params>(
|
|
|
25
25
|
const { input, step, ctx, runId } = args;
|
|
26
26
|
// One step for BOTH facts, because both must survive a replay: a re-resolved audience would
|
|
27
27
|
// deliver to whoever subscribed during the wait, and a re-read clock would write a different
|
|
28
|
-
// `at` into the inbox on every attempt. Epoch ms rather than a `Date
|
|
29
|
-
//
|
|
28
|
+
// `at` into the inbox on every attempt. Epoch ms rather than a `Date`: a step answers the stored
|
|
29
|
+
// form, where a `Date` is a string.
|
|
30
30
|
const open = await step.run('open', async () => ({
|
|
31
31
|
at: ctx.now().getTime(),
|
|
32
32
|
recipients: input.recipients ?? [...(await plan.recipientsFor({ input: input.params, ctx }))],
|
package/src/fcm.ts
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
// Single responsibility: one push to one Android device over FCM HTTP v1 — a service-account
|
|
2
|
+
// assertion (RS256) exchanged for an OAuth access token, cached until shortly before it expires,
|
|
3
|
+
// then the send, each under a deadline, and the answer read as delivered, gone or refused.
|
|
4
|
+
|
|
5
|
+
import { ConfigInvalidError } from '@ultimat3/core';
|
|
6
|
+
import { outbound } from '@ultimat3/core/outbound';
|
|
7
|
+
import { t } from '@ultimat3/schema';
|
|
8
|
+
import { signJwt } from './native-jwt';
|
|
9
|
+
import type { NativeTransportDeps, ProviderRefusal } from './native-transport';
|
|
10
|
+
import { refusalOf, signingKeyFrom } from './native-transport';
|
|
11
|
+
import {
|
|
12
|
+
NotifyPushDeviceGoneError,
|
|
13
|
+
NotifyPushProviderRefusedError,
|
|
14
|
+
providerReason,
|
|
15
|
+
} from './push-errors';
|
|
16
|
+
import type { NativePush, NativeSender } from './push-message';
|
|
17
|
+
import type { TokenCache } from './token-cache';
|
|
18
|
+
import { tokenCache } from './token-cache';
|
|
19
|
+
|
|
20
|
+
export interface FcmOptions {
|
|
21
|
+
/** The Firebase project id — the `project_id` of the service-account JSON. */
|
|
22
|
+
readonly projectId: string;
|
|
23
|
+
/** The service account's `client_email`. Its `private_key` is `ULTIMATE_FCM_PRIVATE_KEY`. */
|
|
24
|
+
readonly clientEmail: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export const GOOGLE_TOKEN_URL = 'https://oauth2.googleapis.com/token';
|
|
28
|
+
export const FCM_ORIGIN = 'https://fcm.googleapis.com';
|
|
29
|
+
export const FCM_SCOPE = 'https://www.googleapis.com/auth/firebase.messaging';
|
|
30
|
+
|
|
31
|
+
/** The assertion's own life (Google's maximum), and how early a cached access token is replaced. */
|
|
32
|
+
const ASSERTION_SECONDS = 3_600;
|
|
33
|
+
export const FCM_TOKEN_REFRESH_BEFORE_MS = 5 * 60_000;
|
|
34
|
+
|
|
35
|
+
const PROJECT = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/;
|
|
36
|
+
const CLIENT_EMAIL = /^[A-Za-z0-9._-]{1,128}@[a-z0-9.-]{1,128}$/;
|
|
37
|
+
|
|
38
|
+
const AccessToken = t.object({ access_token: t.string.min(1), expires_in: t.number.int() });
|
|
39
|
+
|
|
40
|
+
function screen(options: FcmOptions): void {
|
|
41
|
+
for (const [field, value, rule] of [
|
|
42
|
+
['projectId', options.projectId, PROJECT],
|
|
43
|
+
['clientEmail', options.clientEmail, CLIENT_EMAIL],
|
|
44
|
+
] as const) {
|
|
45
|
+
if (!rule.test(value)) {
|
|
46
|
+
throw new ConfigInvalidError({
|
|
47
|
+
cause: `nativePush({ fcm }) ${field} is not what a Firebase service account carries`,
|
|
48
|
+
fix: `nativePush({ fcm: { projectId, clientEmail } }) # copy project_id and client_email from the service-account JSON`,
|
|
49
|
+
meta: { field },
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The `message` the wire carries, built from `push` and nothing else. */
|
|
56
|
+
export function fcmMessage(token: string, push: NativePush): Record<string, unknown> {
|
|
57
|
+
if (push.kind === 'data') {
|
|
58
|
+
// No `notification` block: Android hands the ids to the app and draws nothing.
|
|
59
|
+
return { token, data: { ...push.data }, android: { priority: 'high' } };
|
|
60
|
+
}
|
|
61
|
+
const { push: alert } = push;
|
|
62
|
+
return {
|
|
63
|
+
token,
|
|
64
|
+
notification: { title: alert.title, body: alert.body },
|
|
65
|
+
data: { url: alert.url },
|
|
66
|
+
android: {
|
|
67
|
+
priority: alert.urgent ? 'high' : 'normal',
|
|
68
|
+
ttl: `${String(alert.ttlSeconds)}s`,
|
|
69
|
+
...(alert.collapse === undefined ? {} : { collapse_key: alert.collapse }),
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* FCM's dead-token answer: `UNREGISTERED` in the error's details. Never a bare 404 — a wrong
|
|
76
|
+
* `projectId` is a 404 too, and reading it as "gone" would delete every Android device the app
|
|
77
|
+
* has. The detail is cut at 200 characters, so a body too long to parse is searched as text.
|
|
78
|
+
*/
|
|
79
|
+
function isUnregistered(refusal: ProviderRefusal): boolean {
|
|
80
|
+
const details = (refusal.body?.['error'] as { details?: unknown } | undefined)?.details;
|
|
81
|
+
if (Array.isArray(details)) {
|
|
82
|
+
return details.some(
|
|
83
|
+
(entry: unknown) => (entry as { errorCode?: unknown } | null)?.errorCode === 'UNREGISTERED',
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
return refusal.body === undefined && refusal.detail.includes('"UNREGISTERED"');
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const reasonOf = (body: Readonly<Record<string, unknown>> | undefined): string => {
|
|
90
|
+
const error = body?.['error'];
|
|
91
|
+
// The send API nests `{ error: { status } }`; the token endpoint answers `{ error: 'invalid_grant' }`.
|
|
92
|
+
return providerReason(
|
|
93
|
+
typeof error === 'string' ? error : (error as { status?: unknown })?.status,
|
|
94
|
+
);
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** The sentence FCM answers an unparseable token with — inside the first 200 characters. */
|
|
98
|
+
const BAD_TOKEN_TEXT = 'not a valid FCM registration token';
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* `INVALID_ARGUMENT` is a bad payload as often as a bad token, so only the `BadRequest` violation
|
|
102
|
+
* on `message.token` (or, in a detail cut too short to parse, FCM's own sentence for it) counts
|
|
103
|
+
* against the row: a payload bug must never delete every Android device the app has.
|
|
104
|
+
*/
|
|
105
|
+
function isBadToken(refusal: ProviderRefusal): boolean {
|
|
106
|
+
if (refusal.status !== 400) return false;
|
|
107
|
+
const details = (refusal.body?.['error'] as { details?: unknown } | undefined)?.details;
|
|
108
|
+
if (!Array.isArray(details)) {
|
|
109
|
+
return refusal.body === undefined && refusal.detail.includes(BAD_TOKEN_TEXT);
|
|
110
|
+
}
|
|
111
|
+
return details.some((entry: unknown) => {
|
|
112
|
+
const violations = (entry as { fieldViolations?: unknown } | null)?.fieldViolations;
|
|
113
|
+
return (
|
|
114
|
+
Array.isArray(violations) &&
|
|
115
|
+
violations.some(
|
|
116
|
+
(violation: unknown) =>
|
|
117
|
+
(violation as { field?: unknown } | null)?.field === 'message.token',
|
|
118
|
+
)
|
|
119
|
+
);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const refused = (
|
|
124
|
+
status: number,
|
|
125
|
+
reason: string,
|
|
126
|
+
flags: { credential?: boolean; badToken?: boolean } = {},
|
|
127
|
+
) => new NotifyPushProviderRefusedError({ provider: 'fcm', status, reason, ...flags });
|
|
128
|
+
|
|
129
|
+
export function fcmSender(options: FcmOptions, deps: NativeTransportDeps): NativeSender {
|
|
130
|
+
screen(options);
|
|
131
|
+
const wallMs = (): number => deps.clock.now().getTime();
|
|
132
|
+
const now = (): number => deps.clock.monotonic();
|
|
133
|
+
const access: TokenCache = tokenCache(
|
|
134
|
+
async () => {
|
|
135
|
+
const issuedAt = Math.floor(wallMs() / 1000);
|
|
136
|
+
const key = await signingKeyFrom(deps.env, 'fcm');
|
|
137
|
+
const assertion = await signJwt(
|
|
138
|
+
'RS256',
|
|
139
|
+
key,
|
|
140
|
+
{ typ: 'JWT' },
|
|
141
|
+
{
|
|
142
|
+
iss: options.clientEmail,
|
|
143
|
+
scope: FCM_SCOPE,
|
|
144
|
+
aud: GOOGLE_TOKEN_URL,
|
|
145
|
+
iat: issuedAt,
|
|
146
|
+
exp: issuedAt + ASSERTION_SECONDS,
|
|
147
|
+
},
|
|
148
|
+
);
|
|
149
|
+
try {
|
|
150
|
+
const answer = await outbound(
|
|
151
|
+
{
|
|
152
|
+
url: GOOGLE_TOKEN_URL,
|
|
153
|
+
vendor: 'google-oauth',
|
|
154
|
+
dial: 'named',
|
|
155
|
+
deadlineMs: deps.deadlineMs,
|
|
156
|
+
method: 'POST',
|
|
157
|
+
headers: { 'content-type': 'application/x-www-form-urlencoded' },
|
|
158
|
+
body: new URLSearchParams({
|
|
159
|
+
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
|
160
|
+
assertion,
|
|
161
|
+
}).toString(),
|
|
162
|
+
},
|
|
163
|
+
{ fetch: deps.fetch, now },
|
|
164
|
+
);
|
|
165
|
+
const token = answer.json(AccessToken);
|
|
166
|
+
return { value: token.access_token, expiresAt: wallMs() + token.expires_in * 1000 };
|
|
167
|
+
} catch (error) {
|
|
168
|
+
const refusal = refusalOf(error);
|
|
169
|
+
if (refusal === undefined) throw error;
|
|
170
|
+
// Whatever the status, Google refusing the assertion is the service account's key.
|
|
171
|
+
throw refused(refusal.status, reasonOf(refusal.body), { credential: true });
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
wallMs,
|
|
175
|
+
FCM_TOKEN_REFRESH_BEFORE_MS,
|
|
176
|
+
);
|
|
177
|
+
const url = `${FCM_ORIGIN}/v1/projects/${options.projectId}/messages:send`;
|
|
178
|
+
return {
|
|
179
|
+
async send(token, push, signal) {
|
|
180
|
+
const body = JSON.stringify({ message: fcmMessage(token, push) });
|
|
181
|
+
const bearer = await access.get();
|
|
182
|
+
try {
|
|
183
|
+
await outbound(
|
|
184
|
+
{
|
|
185
|
+
url,
|
|
186
|
+
vendor: 'fcm',
|
|
187
|
+
dial: 'named',
|
|
188
|
+
deadlineMs: deps.deadlineMs,
|
|
189
|
+
method: 'POST',
|
|
190
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${bearer}` },
|
|
191
|
+
body,
|
|
192
|
+
signal,
|
|
193
|
+
},
|
|
194
|
+
{ fetch: deps.fetch, now },
|
|
195
|
+
);
|
|
196
|
+
} catch (error) {
|
|
197
|
+
const refusal = refusalOf(error);
|
|
198
|
+
if (refusal === undefined) throw error;
|
|
199
|
+
if (isUnregistered(refusal)) {
|
|
200
|
+
throw new NotifyPushDeviceGoneError({
|
|
201
|
+
provider: 'fcm',
|
|
202
|
+
status: refusal.status,
|
|
203
|
+
reason: 'UNREGISTERED',
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (refusal.status === 401 || refusal.status === 403) access.invalidate(bearer);
|
|
207
|
+
throw refused(refusal.status, reasonOf(refusal.body), { badToken: isBadToken(refusal) });
|
|
208
|
+
}
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
/** Re-exported so a `notifier` file needs one import, not two. Same object as schema's. */
|
|
8
8
|
export type { Infer } from '@ultimat3/schema';
|
|
9
9
|
export { t } from '@ultimat3/schema';
|
|
10
|
+
export type { ApnsOptions } from './apns';
|
|
10
11
|
export type { AttemptInput } from './attempt';
|
|
11
12
|
export { attemptDelivery } from './attempt';
|
|
12
13
|
export type {
|
|
@@ -21,7 +22,13 @@ export type { InAppChannelOptions } from './channel-in-app';
|
|
|
21
22
|
export { IN_APP_CHANNEL, inAppChannel } from './channel-in-app';
|
|
22
23
|
export type { MailChannelOptions, Mailer, NotifyMail } from './channel-mail';
|
|
23
24
|
export { MAIL_CHANNEL, mailChannel } from './channel-mail';
|
|
24
|
-
export type {
|
|
25
|
+
export type {
|
|
26
|
+
DataPushChannelOptions,
|
|
27
|
+
DataPusher,
|
|
28
|
+
NotifyPushMessage,
|
|
29
|
+
PushChannelOptions,
|
|
30
|
+
Pusher,
|
|
31
|
+
} from './channel-push';
|
|
25
32
|
export { PUSH_CHANNEL, pushChannel } from './channel-push';
|
|
26
33
|
export type {
|
|
27
34
|
DigestAppend,
|
|
@@ -47,6 +54,7 @@ export {
|
|
|
47
54
|
NotifyFanoutTooWideError,
|
|
48
55
|
NotifyStoreMissingError,
|
|
49
56
|
} from './errors';
|
|
57
|
+
export type { FcmOptions } from './fcm';
|
|
50
58
|
// `runFanout` is deliberately absent, for the reason `registerJob` is absent from
|
|
51
59
|
// @ultimat3/jobs: a second way to execute a fan-out would bypass the job the factory built, and
|
|
52
60
|
// with it the retry policy, the cancellation and the manifest row.
|
|
@@ -59,6 +67,7 @@ export {
|
|
|
59
67
|
} from './inbox-pg';
|
|
60
68
|
export type {
|
|
61
69
|
DeliveryClaim,
|
|
70
|
+
DeliveryClaimAnswer,
|
|
62
71
|
DeliveryLedger,
|
|
63
72
|
DeliveryRecord,
|
|
64
73
|
DeliveryStatus,
|
|
@@ -66,6 +75,7 @@ export type {
|
|
|
66
75
|
MemoryDeliveryLedgerOptions,
|
|
67
76
|
} from './ledger';
|
|
68
77
|
export {
|
|
78
|
+
DEFAULT_DELIVERY_RECLAIM_MS,
|
|
69
79
|
DELIVERY_STATUSES,
|
|
70
80
|
isDeliveryStatus,
|
|
71
81
|
memoryDeliveryLedger,
|
|
@@ -75,6 +85,13 @@ export {
|
|
|
75
85
|
postgresDeliveryLedger,
|
|
76
86
|
SQL_NOTIFY_DELIVERIES_TABLE,
|
|
77
87
|
} from './ledger-pg';
|
|
88
|
+
export type { NativePusher, NativePushOptions } from './native-push';
|
|
89
|
+
export {
|
|
90
|
+
DEFAULT_NATIVE_PUSH_DEADLINE_MS,
|
|
91
|
+
nativePush,
|
|
92
|
+
PUSH_BAD_TOKEN_LIMIT,
|
|
93
|
+
} from './native-push';
|
|
94
|
+
export { APNS_KEY_ENV, FCM_KEY_ENV } from './native-transport';
|
|
78
95
|
export type { NotifyEvent, Recipient } from './notification';
|
|
79
96
|
export { recipientSchema } from './notification';
|
|
80
97
|
export type { NotifierDefinition } from './notifier';
|
|
@@ -93,6 +110,24 @@ export type {
|
|
|
93
110
|
export { toDurationMs } from './plan';
|
|
94
111
|
export type { MemoryPreferenceStore, PreferenceQuery, PreferenceStore } from './preferences';
|
|
95
112
|
export { allowAllPreferences, memoryPreferenceStore } from './preferences';
|
|
113
|
+
export type { PushDeviceRecord, PushDeviceStore, PushPlatform } from './push-device';
|
|
114
|
+
export { memoryPushDeviceStore, PUSH_PLATFORMS } from './push-device';
|
|
115
|
+
export type { PostgresPushDeviceStoreOptions } from './push-device-pg';
|
|
116
|
+
export {
|
|
117
|
+
PUSH_DEVICES_TABLE,
|
|
118
|
+
postgresPushDeviceStore,
|
|
119
|
+
SQL_PUSH_DEVICES_TABLE,
|
|
120
|
+
} from './push-device-pg';
|
|
121
|
+
export type { PushProvider } from './push-errors';
|
|
122
|
+
export {
|
|
123
|
+
NotifyPushDeviceGoneError,
|
|
124
|
+
NotifyPushMessageInvalidError,
|
|
125
|
+
NotifyPushProviderRefusedError,
|
|
126
|
+
} from './push-errors';
|
|
127
|
+
export type { PushData } from './push-message';
|
|
128
|
+
export { DATA_PUSH_MAX_KEYS, DATA_PUSH_MAX_VALUE } from './push-message';
|
|
129
|
+
export type { PushRegisterOptions } from './push-register';
|
|
130
|
+
export { PUSH_DEVICES_PER_ACTOR, pushRegister, pushUnregister } from './push-register';
|
|
96
131
|
export { purgeNotifyDeliveries, purgeNotifyDigests, purgeNotifyInbox } from './retention';
|
|
97
132
|
export type { InstalledNotifyStores, NotifyStores } from './stores';
|
|
98
133
|
export {
|
package/src/ledger-pg.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
// Statements are spelled out so an agent can run the exact one it saw in a log.
|
|
6
6
|
|
|
7
7
|
import type { PgExecutor } from '@ultimat3/core';
|
|
8
|
-
import { finiteCount } from '@ultimat3/core';
|
|
8
|
+
import { finiteCount, uuidV7 } from '@ultimat3/core';
|
|
9
9
|
import type { DeliveryClaim, DeliveryLedger, DeliveryRecord, DeliveryStatus } from './ledger';
|
|
10
|
-
import { isDeliveryStatus } from './ledger';
|
|
10
|
+
import { DEFAULT_DELIVERY_RECLAIM_MS, isDeliveryStatus } from './ledger';
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Applied the way `SQL_JOBS_TABLE` and `SQL_IDEMPOTENCY_TABLE` are — by the boot, not by an app
|
|
@@ -36,25 +36,40 @@ create unique index if not exists x_notify_deliveries_claim_idx
|
|
|
36
36
|
on x_notify_deliveries (notifier, key, channel, coalesce(recipient, ''));
|
|
37
37
|
|
|
38
38
|
create index if not exists x_notify_deliveries_at_idx on x_notify_deliveries (at);
|
|
39
|
+
|
|
40
|
+
alter table x_notify_deliveries add column if not exists claim_id uuid;
|
|
39
41
|
`;
|
|
40
42
|
|
|
41
43
|
/**
|
|
42
|
-
* The claim, atomic in one statement. The `do update` fires
|
|
43
|
-
*
|
|
44
|
-
* means
|
|
44
|
+
* The claim, atomic in one statement. The `do update` fires for a row that is not `sent` and not a
|
|
45
|
+
* live claim, so a returned row always means this caller owns the delivery and must send. No row
|
|
46
|
+
* back means it went out, or another attempt holds it: `SQL_NOTIFY_FIND` says which.
|
|
47
|
+
*
|
|
48
|
+
* `$7` is the job's clock minus the reclaim window, the rule `purgeExpired` states: `at` was written
|
|
49
|
+
* by whichever process took the delivery, so `now()` here would measure two clocks' offset.
|
|
45
50
|
*/
|
|
46
51
|
export const SQL_NOTIFY_CLAIM = `
|
|
47
|
-
insert into x_notify_deliveries (notifier, key, recipient, channel, status, attempts, at)
|
|
48
|
-
values ($1, $2, $3, $4, 'sending', 1, $5)
|
|
52
|
+
insert into x_notify_deliveries (notifier, key, recipient, channel, status, attempts, at, claim_id)
|
|
53
|
+
values ($1, $2, $3, $4, 'sending', 1, $5, $6::uuid)
|
|
49
54
|
on conflict (notifier, key, channel, coalesce(recipient, ''))
|
|
50
|
-
do update set attempts = x_notify_deliveries.attempts + 1, status = 'sending', at = excluded.at
|
|
55
|
+
do update set attempts = x_notify_deliveries.attempts + 1, status = 'sending', at = excluded.at,
|
|
56
|
+
claim_id = excluded.claim_id
|
|
51
57
|
where x_notify_deliveries.status <> 'sent'
|
|
58
|
+
and (x_notify_deliveries.status <> 'sending' or x_notify_deliveries.at < $7)
|
|
52
59
|
returning attempts
|
|
53
60
|
`;
|
|
54
61
|
|
|
62
|
+
/**
|
|
63
|
+
* `claim_id = $7 and status = 'sending'` is a FENCE, as on `SQL_IDEMPOTENCY_SETTLE`, and BOTH halves
|
|
64
|
+
* for the same reason: the id alone lets an attempt overwrite its own settlement, the status alone
|
|
65
|
+
* lets a straggler settle the row its reclaimer is sending. A row an older version left `sending`
|
|
66
|
+
* has no id, so nothing but a fresh claim settles it. `returning key` makes the refusal observable.
|
|
67
|
+
*/
|
|
55
68
|
export const SQL_NOTIFY_SETTLE = `
|
|
56
69
|
update x_notify_deliveries set status = $5, at = $6
|
|
57
70
|
where notifier = $1 and key = $2 and channel = $4 and coalesce(recipient, '') = coalesce($3, '')
|
|
71
|
+
and claim_id = $7::uuid and status = 'sending'
|
|
72
|
+
returning key
|
|
58
73
|
`;
|
|
59
74
|
|
|
60
75
|
export const SQL_NOTIFY_FIND = `
|
|
@@ -114,6 +129,8 @@ export interface PostgresDeliveryLedgerOptions {
|
|
|
114
129
|
* than by two defaults that happen to agree.
|
|
115
130
|
*/
|
|
116
131
|
readonly windowMs?: number | undefined;
|
|
132
|
+
/** How long a `sending` row is held. Defaults to `DEFAULT_DELIVERY_RECLAIM_MS`; longer than your slowest send. */
|
|
133
|
+
readonly reclaimAfterMs?: number | undefined;
|
|
117
134
|
}
|
|
118
135
|
|
|
119
136
|
/** 24 hours — the same default `postgresIdempotencyStore` carries, for the reason above. */
|
|
@@ -140,6 +157,17 @@ export function postgresDeliveryLedger(
|
|
|
140
157
|
options.windowMs ?? DEFAULT_DELIVERY_WINDOW_MS,
|
|
141
158
|
1,
|
|
142
159
|
);
|
|
160
|
+
const reclaimAfterMs = finiteCount(
|
|
161
|
+
'postgresDeliveryLedger',
|
|
162
|
+
'reclaimAfterMs',
|
|
163
|
+
options.reclaimAfterMs ?? DEFAULT_DELIVERY_RECLAIM_MS,
|
|
164
|
+
1,
|
|
165
|
+
);
|
|
166
|
+
const find = async (claim: DeliveryClaim): Promise<DeliveryRecord | undefined> => {
|
|
167
|
+
const rows = await executor.query<DeliveryRow>(SQL_NOTIFY_FIND, argsOf(claim));
|
|
168
|
+
const row = rows[0];
|
|
169
|
+
return row === undefined ? undefined : toRecord(row);
|
|
170
|
+
};
|
|
143
171
|
return {
|
|
144
172
|
windowMs,
|
|
145
173
|
async purgeExpired(nowMs) {
|
|
@@ -153,20 +181,31 @@ export function postgresDeliveryLedger(
|
|
|
153
181
|
return rows.length;
|
|
154
182
|
},
|
|
155
183
|
async claim(claim, at) {
|
|
184
|
+
const claimId = uuidV7();
|
|
156
185
|
const rows = await executor.query<{ attempts: number }>(SQL_NOTIFY_CLAIM, [
|
|
157
186
|
...argsOf(claim),
|
|
158
187
|
at,
|
|
188
|
+
claimId,
|
|
189
|
+
new Date(at.getTime() - reclaimAfterMs),
|
|
159
190
|
]);
|
|
160
|
-
|
|
191
|
+
if (rows.length > 0) return { outcome: 'owned', claimId };
|
|
192
|
+
// Two statements, and safe as two: `sent` never moves, and any other answer only retries. A
|
|
193
|
+
// row a sweep took meanwhile reads as held too, and the retry then claims it clean.
|
|
194
|
+
const found = await find(claim);
|
|
195
|
+
if (found?.status === 'sent') return { outcome: 'sent' };
|
|
196
|
+
const left = (found?.at.getTime() ?? at.getTime()) + reclaimAfterMs - at.getTime();
|
|
197
|
+
return { outcome: 'held', retryAfterMs: Math.max(0, left) };
|
|
161
198
|
},
|
|
162
|
-
async settle(claim, status, at) {
|
|
163
|
-
await executor.query(SQL_NOTIFY_SETTLE, [
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
199
|
+
async settle(claim, status, at, claimId) {
|
|
200
|
+
const rows = await executor.query<{ key: string }>(SQL_NOTIFY_SETTLE, [
|
|
201
|
+
...argsOf(claim),
|
|
202
|
+
status,
|
|
203
|
+
at,
|
|
204
|
+
claimId,
|
|
205
|
+
]);
|
|
206
|
+
return rows.length > 0;
|
|
169
207
|
},
|
|
208
|
+
find,
|
|
170
209
|
};
|
|
171
210
|
}
|
|
172
211
|
|
package/src/ledger.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// replays the send. This is the second layer: the claim is taken atomically before the send, and a
|
|
6
6
|
// claim that already reads `sent` answers `false` and the fan-out skips.
|
|
7
7
|
|
|
8
|
-
import { finiteCount } from '@ultimat3/core';
|
|
8
|
+
import { finiteCount, uuidV7 } from '@ultimat3/core';
|
|
9
9
|
|
|
10
10
|
export const DELIVERY_STATUSES = ['sending', 'sent', 'failed'] as const;
|
|
11
11
|
|
|
@@ -39,21 +39,44 @@ export interface DeliveryRecord extends DeliveryClaim {
|
|
|
39
39
|
readonly at: Date;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
/**
|
|
43
|
+
* What a claim answers, and the three are three because each asks something different of the caller.
|
|
44
|
+
* `owned`: send, and settle under this id. `sent`: it went out, stop. `held`: a live attempt holds
|
|
45
|
+
* it. This attempt must NOT send and must NOT read that as `sent` either, or a holder that then dies
|
|
46
|
+
* is a notification nobody retries. `retryAfterMs` is what is left of the holder's window.
|
|
47
|
+
*/
|
|
48
|
+
export type DeliveryClaimAnswer =
|
|
49
|
+
| { readonly outcome: 'owned'; readonly claimId: string }
|
|
50
|
+
| { readonly outcome: 'sent' }
|
|
51
|
+
| { readonly outcome: 'held'; readonly retryAfterMs: number };
|
|
52
|
+
|
|
42
53
|
export interface DeliveryLedger {
|
|
43
54
|
/**
|
|
44
|
-
* Take ownership of this delivery.
|
|
45
|
-
* exists and this attempt must not send.
|
|
55
|
+
* Take ownership of this delivery. A `failed` row is claimed at once.
|
|
46
56
|
*
|
|
47
|
-
* A row left `sending`
|
|
48
|
-
* notification that silently never arrives because the process that owned it died.
|
|
49
|
-
*
|
|
57
|
+
* A row left `sending` IS re-claimable once its window has passed, deliberately: the alternative
|
|
58
|
+
* is a notification that silently never arrives because the process that owned it died. Inside
|
|
59
|
+
* the window it is `held`, so a second attempt running beside the first does not send too.
|
|
60
|
+
* At-least-once is the guarantee this package offers and the honest one: a send outlasting the
|
|
61
|
+
* window still overlaps its reclaimer.
|
|
50
62
|
*/
|
|
51
|
-
claim(claim: DeliveryClaim, at: Date): Promise<
|
|
52
|
-
/**
|
|
53
|
-
|
|
63
|
+
claim(claim: DeliveryClaim, at: Date): Promise<DeliveryClaimAnswer>;
|
|
64
|
+
/**
|
|
65
|
+
* Record the outcome, if `claimId` still owns the row and it is still `sending`. `false` means it
|
|
66
|
+
* did not, and nothing changed: a late attempt never moves its reclaimer's row, least of all from
|
|
67
|
+
* `sent` back to `failed`. Only `sent` blocks a future claim.
|
|
68
|
+
*/
|
|
69
|
+
settle(claim: DeliveryClaim, status: DeliveryStatus, at: Date, claimId: string): Promise<boolean>;
|
|
54
70
|
find(claim: DeliveryClaim): Promise<DeliveryRecord | undefined>;
|
|
55
71
|
}
|
|
56
72
|
|
|
73
|
+
/**
|
|
74
|
+
* How long a `sending` row belongs to the attempt that claimed it. Five minutes: past the 30 s a job
|
|
75
|
+
* lease survives a dead worker by, and past any send a provider SDK's own timeout allows. Too short
|
|
76
|
+
* is a duplicate send; too long is a crashed attempt's recipient waiting that much longer.
|
|
77
|
+
*/
|
|
78
|
+
export const DEFAULT_DELIVERY_RECLAIM_MS = 5 * 60 * 1000;
|
|
79
|
+
|
|
57
80
|
/**
|
|
58
81
|
* The tuple as one string. `JSON.stringify` of an ARRAY rather than a joined separator: a
|
|
59
82
|
* recipient id or a channel name containing the separator would otherwise collide two distinct
|
|
@@ -70,6 +93,8 @@ export interface MemoryDeliveryLedgerOptions {
|
|
|
70
93
|
* into the heap, and it says so by publishing `size`.
|
|
71
94
|
*/
|
|
72
95
|
readonly max?: number;
|
|
96
|
+
/** How long a `sending` row is held. Defaults to `DEFAULT_DELIVERY_RECLAIM_MS`. */
|
|
97
|
+
readonly reclaimAfterMs?: number | undefined;
|
|
73
98
|
}
|
|
74
99
|
|
|
75
100
|
export const DEFAULT_MAX_DELIVERY_RECORDS = 10_000;
|
|
@@ -100,7 +125,13 @@ export function memoryDeliveryLedger(
|
|
|
100
125
|
options.max ?? DEFAULT_MAX_DELIVERY_RECORDS,
|
|
101
126
|
1,
|
|
102
127
|
);
|
|
103
|
-
const
|
|
128
|
+
const reclaimAfterMs = finiteCount(
|
|
129
|
+
'memoryDeliveryLedger',
|
|
130
|
+
'reclaimAfterMs',
|
|
131
|
+
options.reclaimAfterMs ?? DEFAULT_DELIVERY_RECLAIM_MS,
|
|
132
|
+
1,
|
|
133
|
+
);
|
|
134
|
+
const rows = new Map<string, { readonly record: DeliveryRecord; readonly claimId: string }>();
|
|
104
135
|
let dropped = 0;
|
|
105
136
|
|
|
106
137
|
const evict = (): void => {
|
|
@@ -121,28 +152,37 @@ export function memoryDeliveryLedger(
|
|
|
121
152
|
},
|
|
122
153
|
claim(claim, at) {
|
|
123
154
|
const id = keyOf(claim);
|
|
124
|
-
const existing = rows.get(id);
|
|
125
|
-
if (existing?.status === 'sent') return Promise.resolve(
|
|
155
|
+
const existing = rows.get(id)?.record;
|
|
156
|
+
if (existing?.status === 'sent') return Promise.resolve({ outcome: 'sent' });
|
|
157
|
+
const reclaimAt = (existing?.at.getTime() ?? 0) + reclaimAfterMs;
|
|
158
|
+
if (existing?.status === 'sending' && at.getTime() <= reclaimAt) {
|
|
159
|
+
return Promise.resolve({ outcome: 'held', retryAfterMs: reclaimAt - at.getTime() });
|
|
160
|
+
}
|
|
161
|
+
const claimId = uuidV7();
|
|
126
162
|
// The row it found keeps its own identity, as the pg `on conflict do update` does.
|
|
127
|
-
|
|
163
|
+
const record: DeliveryRecord = {
|
|
128
164
|
...(existing ?? claim),
|
|
129
165
|
status: 'sending',
|
|
130
166
|
attempts: (existing?.attempts ?? 0) + 1,
|
|
131
167
|
at,
|
|
132
|
-
}
|
|
168
|
+
};
|
|
169
|
+
rows.set(id, { record, claimId });
|
|
133
170
|
evict();
|
|
134
|
-
return Promise.resolve(
|
|
171
|
+
return Promise.resolve({ outcome: 'owned', claimId });
|
|
135
172
|
},
|
|
136
|
-
settle(claim, status, at) {
|
|
173
|
+
settle(claim, status, at, claimId) {
|
|
137
174
|
const id = keyOf(claim);
|
|
138
175
|
const existing = rows.get(id);
|
|
139
176
|
// No claim, no row: the pg `update` matches nothing, and a row written here would refuse
|
|
140
177
|
// the claim after it — a notification dev never sends and production does.
|
|
141
|
-
if (existing !==
|
|
142
|
-
|
|
178
|
+
if (existing?.claimId !== claimId || existing.record.status !== 'sending') {
|
|
179
|
+
return Promise.resolve(false);
|
|
180
|
+
}
|
|
181
|
+
rows.set(id, { claimId, record: { ...existing.record, status, at } });
|
|
182
|
+
return Promise.resolve(true);
|
|
143
183
|
},
|
|
144
184
|
find(claim) {
|
|
145
|
-
return Promise.resolve(rows.get(keyOf(claim)));
|
|
185
|
+
return Promise.resolve(rows.get(keyOf(claim))?.record);
|
|
146
186
|
},
|
|
147
187
|
clear() {
|
|
148
188
|
rows.clear();
|