whatsapp_notifier 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/generators/whatsapp_notifier/install_service_generator.rb +2 -0
- data/lib/whatsapp_notifier/client.rb +12 -0
- data/lib/whatsapp_notifier/providers/web_automation.rb +33 -0
- data/lib/whatsapp_notifier/services/web_automation/history.test.ts +695 -0
- data/lib/whatsapp_notifier/services/web_automation/history.ts +323 -0
- data/lib/whatsapp_notifier/services/web_automation/inbound.test.ts +209 -2
- data/lib/whatsapp_notifier/services/web_automation/inbound.ts +138 -16
- data/lib/whatsapp_notifier/services/web_automation/index.ts +81 -12
- data/lib/whatsapp_notifier/services/web_automation/media.test.ts +175 -9
- data/lib/whatsapp_notifier/services/web_automation/media.ts +94 -4
- data/lib/whatsapp_notifier/services/web_automation/send.test.ts +20 -0
- data/lib/whatsapp_notifier/services/web_automation/send.ts +17 -0
- data/lib/whatsapp_notifier/version.rb +1 -1
- data/lib/whatsapp_notifier/web_adapter.rb +85 -5
- data/lib/whatsapp_notifier.rb +12 -0
- data/spec/client_spec.rb +28 -1
- data/spec/generators/install_service_generator_spec.rb +1 -1
- data/spec/providers/web_automation_spec.rb +61 -3
- data/spec/web_adapter_spec.rb +232 -1
- data/spec/whatsapp_notifier_spec.rb +27 -0
- metadata +5 -1
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
// Inbound capture core (v0.4.0)
|
|
1
|
+
// Inbound capture core (v0.4.0; two-way since v0.8.0)
|
|
2
2
|
//
|
|
3
3
|
// Pure, testable logic for the two-way layer. Kept separate from index.ts so
|
|
4
4
|
// it can be unit-tested without booting a whatsapp-web.js Client.
|
|
5
5
|
//
|
|
6
|
-
// Policy: surface real
|
|
7
|
-
// —
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
6
|
+
// Policy: surface real 1:1 messages in BOTH directions (phone @c.us or
|
|
7
|
+
// privacy-id @lid) — customer replies AND messages the operator sends from
|
|
8
|
+
// the WhatsApp app itself (fromMe), so host threads show whole conversations.
|
|
9
|
+
// Groups (@g.us), status broadcasts, and non-text system events
|
|
10
|
+
// (e2e_notification, call_log, revoked, …) that carry no real reply are
|
|
11
|
+
// dropped. Captured messages buffer in an in-memory queue drained by GET
|
|
12
|
+
// /inbound/:userId (at-least-once; the host dedupes on messageId — which also
|
|
13
|
+
// eats the fromMe echo of its own /send calls — and decides relevance by
|
|
14
|
+
// matching the resolved phone to its own recipient records).
|
|
12
15
|
|
|
13
16
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
14
17
|
import { join } from 'path';
|
|
@@ -29,6 +32,11 @@ export interface InboundMsg {
|
|
|
29
32
|
mediaFilename?: string;
|
|
30
33
|
mediaSize?: number;
|
|
31
34
|
senderName?: string;
|
|
35
|
+
// 0.8.0 two-way capture. Present ONLY on operator-sent messages (fromMe),
|
|
36
|
+
// where `to` carries the counterparty (customer) chat id the host threads
|
|
37
|
+
// on. Inbound payloads keep the exact pre-0.8.0 shape.
|
|
38
|
+
fromMe?: boolean;
|
|
39
|
+
to?: string;
|
|
32
40
|
}
|
|
33
41
|
|
|
34
42
|
// Media verdict merged into the payload by captureInbound — structurally
|
|
@@ -113,11 +121,62 @@ export function drainInbound(userId: string): InboundMsg[] {
|
|
|
113
121
|
return q;
|
|
114
122
|
}
|
|
115
123
|
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
124
|
+
// ── Self-send echo registry ──
|
|
125
|
+
//
|
|
126
|
+
// Every POST /send fires its own fromMe message_create echo. Letting that
|
|
127
|
+
// echo run the capture pipeline is pure waste: a media send re-downloads the
|
|
128
|
+
// attachment it just uploaded (up to the 30s download budget on the sending
|
|
129
|
+
// Chromium) and stores bytes nobody will ever fetch against the per-user
|
|
130
|
+
// media cap — at campaign scale those phantom copies churn the cap, evicting
|
|
131
|
+
// REAL customer inbound media the operator still wants. The echo's payload is
|
|
132
|
+
// redundant too: the /send response already handed the host this exact
|
|
133
|
+
// messageId, and the host's id-dedupe would discard the echo anyway — so a
|
|
134
|
+
// registry hit is suppressed ENTIRELY (no media resolution, no enqueue, no
|
|
135
|
+
// webhook), which also closes the host-side echo-beats-response adopt race
|
|
136
|
+
// for same-process sends. Messages NOT in the registry (typed on the phone,
|
|
137
|
+
// or replayed by the reconnect backfill) flow through unchanged.
|
|
138
|
+
//
|
|
139
|
+
// Best-effort by design: the registry is in-memory, so an echo landing after
|
|
140
|
+
// a service restart (or one that fires before /send finishes registering the
|
|
141
|
+
// id) flows through — the host's id-dedupe catches it, harmless. Bounded per
|
|
142
|
+
// user and TTL'd because echoes land within seconds of the send; the limits
|
|
143
|
+
// only have to outlive the echo lag, not the conversation.
|
|
144
|
+
export const SELF_SEND_MAX = 200;
|
|
145
|
+
export const SELF_SEND_TTL_MS = 10 * 60 * 1000;
|
|
146
|
+
|
|
147
|
+
// userId → (messageId → expiry epoch ms). Maps iterate in insertion order,
|
|
148
|
+
// so the first key is always the oldest send — eviction is O(evicted).
|
|
149
|
+
const selfSendIds = new Map<string, Map<string, number>>();
|
|
150
|
+
|
|
151
|
+
export function rememberSelfSend(userId: string, messageId: string, nowMs = Date.now()) {
|
|
152
|
+
let ids = selfSendIds.get(userId);
|
|
153
|
+
if (!ids) { ids = new Map(); selfSendIds.set(userId, ids); }
|
|
154
|
+
ids.set(messageId, nowMs + SELF_SEND_TTL_MS);
|
|
155
|
+
while (ids.size > SELF_SEND_MAX) {
|
|
156
|
+
ids.delete(ids.keys().next().value as string);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function isSelfSend(userId: string, messageId: string, nowMs = Date.now()): boolean {
|
|
161
|
+
const ids = selfSendIds.get(userId);
|
|
162
|
+
const expiry = ids && ids.get(messageId);
|
|
163
|
+
if (expiry === undefined) return false;
|
|
164
|
+
if (nowMs > expiry) {
|
|
165
|
+
ids!.delete(messageId); // expired — forget it so the map stays lean
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Sanity filter for a real 1:1 message in either direction. Accepts both
|
|
172
|
+
// phone-number chats (@c.us) and privacy-id chats (@lid — newer WhatsApp
|
|
173
|
+
// delivers replies from an @lid). The jid gate always validates the
|
|
174
|
+
// COUNTERPARTY: the customer is at `from` on inbound but at `to` on
|
|
175
|
+
// operator-sent (fromMe) messages — gating fromMe on `from` (the operator's
|
|
176
|
+
// own jid, always @c.us) would let group/status posts through. Groups
|
|
177
|
+
// (@g.us) and status (@broadcast) are dropped. The host app decides relevance
|
|
178
|
+
// by matching the resolved phone to its own records, so we no longer gate on
|
|
179
|
+
// the per-send allowlist (which was unreliable).
|
|
121
180
|
// Real human/content message types. Anything else (e2e_notification,
|
|
122
181
|
// notification_template, call_log, revoked, protocol, gp2, …) is a system event
|
|
123
182
|
// with no body and must NOT be surfaced as a reply.
|
|
@@ -126,9 +185,9 @@ const TEXTUAL_TYPES = new Set([
|
|
|
126
185
|
]);
|
|
127
186
|
|
|
128
187
|
export function shouldCapture(userId: string, msg: any): boolean {
|
|
129
|
-
if (!msg
|
|
130
|
-
const
|
|
131
|
-
if (!
|
|
188
|
+
if (!msg) return false;
|
|
189
|
+
const counterparty: string = (msg.fromMe ? msg.to : msg.from) || '';
|
|
190
|
+
if (!counterparty.endsWith('@c.us') && !counterparty.endsWith('@lid')) return false;
|
|
132
191
|
if (msg.isStatus) return false;
|
|
133
192
|
if (msg.type && !TEXTUAL_TYPES.has(msg.type)) return false; // drop system events
|
|
134
193
|
return true;
|
|
@@ -136,13 +195,26 @@ export function shouldCapture(userId: string, msg: any): boolean {
|
|
|
136
195
|
|
|
137
196
|
export function normalizeInbound(msg: any, media?: InboundMediaInfo): InboundMsg {
|
|
138
197
|
const from: string = msg.from || '';
|
|
198
|
+
const to: string = msg.to || '';
|
|
199
|
+
// The fallback id keys on the COUNTERPARTY (the customer): on fromMe the
|
|
200
|
+
// `from` is the operator's own jid, shared by every chat — id-less
|
|
201
|
+
// operator messages to different customers in the same second must not
|
|
202
|
+
// collide in the host's dedupe.
|
|
203
|
+
const counterparty = msg.fromMe ? to : from;
|
|
139
204
|
const inbound: InboundMsg = {
|
|
140
205
|
from,
|
|
141
206
|
body: msg.body || '',
|
|
142
|
-
messageId: (msg.id && msg.id._serialized) || `${
|
|
207
|
+
messageId: (msg.id && msg.id._serialized) || `${counterparty}-${msg.timestamp}`,
|
|
143
208
|
timestamp: msg.timestamp || Math.floor(Date.now() / 1000),
|
|
144
209
|
type: msg.type || 'chat'
|
|
145
210
|
};
|
|
211
|
+
// fromMe/to are added ONLY for operator-sent messages — like the media
|
|
212
|
+
// keys below, inbound payloads keep the exact pre-0.8.0 shape so older
|
|
213
|
+
// hosts stay byte-compatible and newer ones key-gate on fromMe presence.
|
|
214
|
+
if (msg.fromMe) {
|
|
215
|
+
inbound.fromMe = true;
|
|
216
|
+
inbound.to = to;
|
|
217
|
+
}
|
|
146
218
|
// Media keys are added ONLY for media messages so text payloads keep the
|
|
147
219
|
// exact 0.6.0 five-field shape (hosts key-gate on hasMedia presence).
|
|
148
220
|
if (msg.hasMedia) inbound.hasMedia = true;
|
|
@@ -165,6 +237,10 @@ export interface CaptureDeps {
|
|
|
165
237
|
export async function processInbound(userId: string, msg: any, deps: CaptureDeps) {
|
|
166
238
|
if (!shouldCapture(userId, msg)) return;
|
|
167
239
|
|
|
240
|
+
// Operator-sent messages take their own (shorter) path: no sender to
|
|
241
|
+
// resolve, and the chat is keyed by the counterparty at msg.to.
|
|
242
|
+
if (msg.fromMe) return processOwnMessage(userId, msg, deps);
|
|
243
|
+
|
|
168
244
|
// One best-effort contact lookup feeds both the sender's display name
|
|
169
245
|
// and the @lid phone resolution. Failure must never drop the message —
|
|
170
246
|
// unless the sender is an unresolvable @lid (handled below).
|
|
@@ -212,6 +288,48 @@ export async function processInbound(userId: string, msg: any, deps: CaptureDeps
|
|
|
212
288
|
if (deps.push) deps.push(userId, inbound);
|
|
213
289
|
}
|
|
214
290
|
|
|
291
|
+
// Operator-sent (fromMe) leg of the pipeline. The senderName contact lookup
|
|
292
|
+
// is skipped on purpose: the "sender" is the operator themself, so the name
|
|
293
|
+
// adds nothing and the lookup costs a puppeteer roundtrip per message. Media
|
|
294
|
+
// resolution, enqueue and webhook are the SAME as inbound — operator photos/
|
|
295
|
+
// documents sync through the existing GET /media path under the same caps.
|
|
296
|
+
async function processOwnMessage(userId: string, msg: any, deps: CaptureDeps) {
|
|
297
|
+
// The echo of our own /send: suppress entirely — no media resolution, no
|
|
298
|
+
// enqueue, no webhook (see the self-send registry above for why).
|
|
299
|
+
// rememberTarget is skipped too: /send already allowlisted this recipient.
|
|
300
|
+
const messageId = msg.id && msg.id._serialized;
|
|
301
|
+
if (messageId && isSelfSend(userId, messageId)) return;
|
|
302
|
+
|
|
303
|
+
const to: string = msg.to || '';
|
|
304
|
+
// An @lid counterparty carries no phone number the host can thread on,
|
|
305
|
+
// and unlike inbound there is no contact handle to resolve it through
|
|
306
|
+
// (msg.getContact() resolves the SENDER — here, the operator). Rare in
|
|
307
|
+
// practice: operator-initiated chats are keyed by the phone @c.us. Drop
|
|
308
|
+
// with a log rather than forward an unmatchable body.
|
|
309
|
+
if (to.endsWith('@lid')) {
|
|
310
|
+
console.log(`Dropping fromMe message to unresolved @lid chat for ${userId}`);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// Same kept-message-earns-the-download rule as inbound: every resolver
|
|
315
|
+
// failure mode reports 'unavailable' instead of throwing.
|
|
316
|
+
let media: InboundMediaInfo | undefined;
|
|
317
|
+
if (msg.hasMedia) {
|
|
318
|
+
media = await deps.resolveMedia(userId, msg);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const inbound = normalizeInbound(msg, media);
|
|
322
|
+
|
|
323
|
+
// A fromMe message to a brand-new number means the operator opened the
|
|
324
|
+
// conversation in the WhatsApp app — allowlist the chat exactly like
|
|
325
|
+
// /send does for its recipients, so the reconnect backfill can replay
|
|
326
|
+
// this conversation after a disconnect window too.
|
|
327
|
+
rememberTarget(userId, to);
|
|
328
|
+
|
|
329
|
+
enqueueInbound(userId, inbound);
|
|
330
|
+
if (deps.push) deps.push(userId, inbound);
|
|
331
|
+
}
|
|
332
|
+
|
|
215
333
|
// Minimal slice of whatsapp-web.js Client that backfill needs — a seam so the
|
|
216
334
|
// replay loop can be tested without booting a real client.
|
|
217
335
|
export interface ChatLike {
|
|
@@ -267,10 +385,14 @@ export async function backfillTargets(
|
|
|
267
385
|
export function clearInbound(userId: string) {
|
|
268
386
|
inboundQueues.delete(userId);
|
|
269
387
|
outboundTargets.delete(userId);
|
|
388
|
+
// Self-send echo ids belong to the old pairing too — and suppression must
|
|
389
|
+
// never leak across a re-pair (however unlikely an id collision is).
|
|
390
|
+
selfSendIds.delete(userId);
|
|
270
391
|
}
|
|
271
392
|
|
|
272
393
|
// Test helper: wipe in-memory state between examples.
|
|
273
394
|
export function resetInboundState() {
|
|
274
395
|
inboundQueues.clear();
|
|
275
396
|
outboundTargets.clear();
|
|
397
|
+
selfSendIds.clear();
|
|
276
398
|
}
|
|
@@ -16,11 +16,14 @@ import {
|
|
|
16
16
|
InboundMsg,
|
|
17
17
|
configureInbound,
|
|
18
18
|
rememberTarget,
|
|
19
|
+
rememberSelfSend,
|
|
19
20
|
backfillTargets,
|
|
21
|
+
resolveChat,
|
|
20
22
|
drainInbound,
|
|
21
23
|
clearInbound,
|
|
22
24
|
processInbound
|
|
23
25
|
} from './inbound';
|
|
26
|
+
import { chatsResponse, historyResponse, refetchResponse, HistoryDeps, RefetchDeps } from './history';
|
|
24
27
|
import {
|
|
25
28
|
configureMedia,
|
|
26
29
|
resolveMediaForMessage,
|
|
@@ -29,6 +32,7 @@ import {
|
|
|
29
32
|
mediaGetResponse,
|
|
30
33
|
mediaDeleteResponse
|
|
31
34
|
} from './media';
|
|
35
|
+
import { sentMessageId } from './send';
|
|
32
36
|
|
|
33
37
|
const app = new Hono();
|
|
34
38
|
const port = Number(process.env.PORT || 3001);
|
|
@@ -114,8 +118,10 @@ async function captureInbound(userId: string, msg: any) {
|
|
|
114
118
|
async function backfillInbound(userId: string, client: Client) {
|
|
115
119
|
// On reconnect, replay recent messages ONLY from chats we actually messaged
|
|
116
120
|
// (the per-send allowlist) so a disconnect window doesn't drop a reply —
|
|
117
|
-
// without scraping every personal conversation on the linked number.
|
|
118
|
-
//
|
|
121
|
+
// without scraping every personal conversation on the linked number.
|
|
122
|
+
// fetchMessages returns BOTH directions, so operator-app (fromMe) messages
|
|
123
|
+
// typed during the window recover too. Live traffic is covered by the
|
|
124
|
+
// message_create handler; this is just recovery.
|
|
119
125
|
await backfillTargets(userId, client, captureInbound);
|
|
120
126
|
}
|
|
121
127
|
|
|
@@ -195,6 +201,8 @@ async function waitForClientReady(clientData: ClientData, timeoutMs = 30000): Pr
|
|
|
195
201
|
throw new Error('Client not ready: WhatsApp Web store did not initialize in time');
|
|
196
202
|
}
|
|
197
203
|
|
|
204
|
+
// Resolves to the sent whatsapp-web.js Message so /send can hand the real
|
|
205
|
+
// message id back to the host (the echo-dedupe key for two-way capture).
|
|
198
206
|
async function sendMessageWithRetry(client: Client, clientData: ClientData, chatId: string, message: string, mediaUrl?: string | null) {
|
|
199
207
|
const maxAttempts = 5;
|
|
200
208
|
|
|
@@ -206,12 +214,10 @@ async function sendMessageWithRetry(client: Client, clientData: ClientData, chat
|
|
|
206
214
|
if (mediaUrl) {
|
|
207
215
|
const { MessageMedia } = require('whatsapp-web.js');
|
|
208
216
|
const media = await MessageMedia.fromUrl(mediaUrl);
|
|
209
|
-
await client.sendMessage(chatId, media, { caption: message });
|
|
210
|
-
} else {
|
|
211
|
-
await client.sendMessage(chatId, message);
|
|
217
|
+
return await client.sendMessage(chatId, media, { caption: message });
|
|
212
218
|
}
|
|
213
219
|
|
|
214
|
-
return;
|
|
220
|
+
return await client.sendMessage(chatId, message);
|
|
215
221
|
} catch (error) {
|
|
216
222
|
console.error(`Send attempt ${attempt}/${maxAttempts} failed for chat ${chatId}:`, error);
|
|
217
223
|
|
|
@@ -302,10 +308,13 @@ async function getOrCreateClient(userId: string): Promise<ClientData> {
|
|
|
302
308
|
backfillInbound(userId, client).catch((e) => console.error(`Backfill failed for ${userId}`, e));
|
|
303
309
|
});
|
|
304
310
|
|
|
305
|
-
// Capture
|
|
306
|
-
// every message across linked/multi-device sessions
|
|
307
|
-
// silently never fires on some)
|
|
308
|
-
//
|
|
311
|
+
// Capture BOTH directions of every 1:1 chat. Only 'message_create' — it
|
|
312
|
+
// fires reliably for every message across linked/multi-device sessions
|
|
313
|
+
// (plain 'message' silently never fires on some) and, unlike 'message',
|
|
314
|
+
// also fires for operator-sent (fromMe) messages, which the host threads
|
|
315
|
+
// as the other half of the conversation since 0.8.0. shouldCapture drops
|
|
316
|
+
// groups/status; the host dedupes on message_id — including the fromMe
|
|
317
|
+
// echo of its own /send calls.
|
|
309
318
|
client.on('message_create', (msg) => captureInbound(userId, msg));
|
|
310
319
|
|
|
311
320
|
client.on('authenticated', () => {
|
|
@@ -537,13 +546,21 @@ app.post('/send/:userId', async (c) => {
|
|
|
537
546
|
|
|
538
547
|
try {
|
|
539
548
|
const chatId = to.includes('@c.us') ? to : `${to}@c.us`;
|
|
540
|
-
await sendMessageWithRetry(data.client, data, chatId, message, mediaUrl);
|
|
549
|
+
const sent = await sendMessageWithRetry(data.client, data, chatId, message, mediaUrl);
|
|
541
550
|
|
|
542
551
|
// Record the recipient so their replies survive a reconnect backfill.
|
|
543
552
|
rememberTarget(userId, chatId);
|
|
544
553
|
|
|
545
554
|
data.lastUsed = Date.now();
|
|
546
|
-
|
|
555
|
+
// messageId is the echo-dedupe key: this send fires its own fromMe
|
|
556
|
+
// message_create, which the host must match by id (see send.ts).
|
|
557
|
+
const messageId = sentMessageId(sent);
|
|
558
|
+
// Register the id so the echo is suppressed service-side (no media
|
|
559
|
+
// re-download, no queue slot, no webhook — see inbound.ts). Best
|
|
560
|
+
// effort: an echo that already fired before sendMessage resolved
|
|
561
|
+
// flows through, and the host's id-dedupe catches it.
|
|
562
|
+
if (messageId) rememberSelfSend(userId, messageId);
|
|
563
|
+
return c.json({ success: true, messageId });
|
|
547
564
|
} catch (error: any) {
|
|
548
565
|
console.error(`Send error for user ${userId}:`, error);
|
|
549
566
|
return c.json({ success: false, error: error.message }, 500);
|
|
@@ -574,6 +591,58 @@ app.get('/media/:userId/:messageId', (c) =>
|
|
|
574
591
|
app.delete('/media/:userId/:messageId', (c) =>
|
|
575
592
|
mediaDeleteResponse(c.req.param('userId'), c.req.param('messageId'), c.req.header('X-WA-Token'), WEBHOOK_TOKEN));
|
|
576
593
|
|
|
594
|
+
// ── Chat history (history.ts) ──
|
|
595
|
+
//
|
|
596
|
+
// Shared deps for the two history routes: the SAME pairing fast-reject and
|
|
597
|
+
// client accessor /send uses (never any other initialization path), plus
|
|
598
|
+
// inbound.ts's chat resolver + reconnect allowlist.
|
|
599
|
+
const historyDeps: HistoryDeps = {
|
|
600
|
+
hasPaired: (userId) => hasPairedSession(userId, clients, sessionDirForUser),
|
|
601
|
+
getClient: getOrCreateClient,
|
|
602
|
+
resolveChat,
|
|
603
|
+
rememberTarget
|
|
604
|
+
};
|
|
605
|
+
|
|
606
|
+
// GET /chats/:userId — list the paired number's 1:1 chats (discovery for the
|
|
607
|
+
// host's old-conversation sync). Token-gated like /media; paired+ready gated
|
|
608
|
+
// like /send; capped at the newest 500 (see history.ts).
|
|
609
|
+
app.get('/chats/:userId', (c) =>
|
|
610
|
+
chatsResponse(c.req.param('userId'), c.req.header('X-WA-Token'), WEBHOOK_TOKEN, historyDeps));
|
|
611
|
+
|
|
612
|
+
// POST /history/:userId { chatId, limit } — replay one chat's history through
|
|
613
|
+
// the live-capture normalizer and return it DIRECTLY in the response (no
|
|
614
|
+
// queue, no webhook — the host ingests synchronously). History media is
|
|
615
|
+
// marked unavailable by design, never downloaded (see history.ts).
|
|
616
|
+
app.post('/history/:userId', async (c) =>
|
|
617
|
+
historyResponse(
|
|
618
|
+
c.req.param('userId'),
|
|
619
|
+
await c.req.json().catch(() => ({})),
|
|
620
|
+
c.req.header('X-WA-Token'),
|
|
621
|
+
WEBHOOK_TOKEN,
|
|
622
|
+
historyDeps
|
|
623
|
+
));
|
|
624
|
+
|
|
625
|
+
// POST /media/:userId/refetch { messageId, chatId } — WhatsApp tap-to-download.
|
|
626
|
+
// The host calls this when an operator opens an evicted/expired media bubble:
|
|
627
|
+
// re-pull THAT message's bytes on demand, store them (cap re-enforced), and
|
|
628
|
+
// report ready so the host can GET /media as usual. Token-gated + paired-ready
|
|
629
|
+
// gated like /history; media no longer upstream → 404 'gone'. Reuses the live
|
|
630
|
+
// resolveMediaForMessage pipeline (per-message size cap + 30s timeout).
|
|
631
|
+
const refetchDeps: RefetchDeps = {
|
|
632
|
+
...historyDeps,
|
|
633
|
+
getMessageById: (client, messageId) => client.getMessageById(messageId),
|
|
634
|
+
resolveMedia: (userId, msg) => resolveMediaForMessage(userId, msg)
|
|
635
|
+
};
|
|
636
|
+
|
|
637
|
+
app.post('/media/:userId/refetch', async (c) =>
|
|
638
|
+
refetchResponse(
|
|
639
|
+
c.req.param('userId'),
|
|
640
|
+
await c.req.json().catch(() => ({})),
|
|
641
|
+
c.req.header('X-WA-Token'),
|
|
642
|
+
WEBHOOK_TOKEN,
|
|
643
|
+
refetchDeps
|
|
644
|
+
));
|
|
645
|
+
|
|
577
646
|
|
|
578
647
|
console.log(`Starting Multi-User WhatsApp service (Bun Native) on port ${port}...`);
|
|
579
648
|
|
|
@@ -16,6 +16,9 @@ import {
|
|
|
16
16
|
mediaTtlMs,
|
|
17
17
|
maxDocumentBytes,
|
|
18
18
|
maxDiskBytes,
|
|
19
|
+
maxUserBytes,
|
|
20
|
+
userDirBytes,
|
|
21
|
+
enforceUserCap,
|
|
19
22
|
downloadPolicy,
|
|
20
23
|
sweepExpired,
|
|
21
24
|
resolveMediaForMessage,
|
|
@@ -29,7 +32,7 @@ const root = mkdtempSync(join(tmpdir(), 'wa-media-'));
|
|
|
29
32
|
let mediaRoot: string;
|
|
30
33
|
let caseId = 0;
|
|
31
34
|
|
|
32
|
-
const ENV_KEYS = ['WHATSAPP_MEDIA_TTL_MS', 'WHATSAPP_MEDIA_MAX_BYTES', 'WHATSAPP_MEDIA_MAX_DISK_BYTES'];
|
|
35
|
+
const ENV_KEYS = ['WHATSAPP_MEDIA_TTL_MS', 'WHATSAPP_MEDIA_MAX_BYTES', 'WHATSAPP_MEDIA_MAX_DISK_BYTES', 'WHATSAPP_MEDIA_MAX_USER_BYTES'];
|
|
33
36
|
const savedEnv: Record<string, string | undefined> = {};
|
|
34
37
|
|
|
35
38
|
beforeEach(() => {
|
|
@@ -282,10 +285,12 @@ test('malformed limit envs fall back to the defaults instead of NaN', () => {
|
|
|
282
285
|
process.env.WHATSAPP_MEDIA_TTL_MS = '2 days';
|
|
283
286
|
process.env.WHATSAPP_MEDIA_MAX_BYTES = 'garbage';
|
|
284
287
|
process.env.WHATSAPP_MEDIA_MAX_DISK_BYTES = '50GB';
|
|
288
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '1 gig';
|
|
285
289
|
|
|
286
290
|
expect(mediaTtlMs()).toBe(48 * 60 * 60 * 1000);
|
|
287
291
|
expect(maxDocumentBytes()).toBe(25 * 1024 * 1024);
|
|
288
292
|
expect(maxDiskBytes()).toBe(5 * 1024 * 1024 * 1024);
|
|
293
|
+
expect(maxUserBytes()).toBe(1024 * 1024 * 1024);
|
|
289
294
|
|
|
290
295
|
// The guards stay live: the document cap still rejects oversize media...
|
|
291
296
|
expect(downloadPolicy('document', 25 * 1024 * 1024 + 1))
|
|
@@ -296,13 +301,151 @@ test('malformed limit envs fall back to the defaults instead of NaN', () => {
|
|
|
296
301
|
expect(sweepExpired(Date.now() + 48 * 60 * 60 * 1000 + 1000)).toBe(1);
|
|
297
302
|
});
|
|
298
303
|
|
|
299
|
-
|
|
300
|
-
|
|
304
|
+
// The per-user cap is now enforced post-write by eviction, so downloadPolicy
|
|
305
|
+
// NEVER skips a download on disk grounds — a user is never starved. Even with
|
|
306
|
+
// a tiny disk cap and a near-full disk, the policy still says download (the
|
|
307
|
+
// post-write enforceUserCap rolls the oldest off instead).
|
|
308
|
+
test('downloadPolicy never refuses on disk grounds (per-user eviction replaces starvation)', () => {
|
|
301
309
|
process.env.WHATSAPP_MEDIA_MAX_DISK_BYTES = '12';
|
|
310
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '12';
|
|
302
311
|
writeMedia(USER, 'taken', bytes('1234567890'), { mime: 'image/png' }); // 10 bytes used
|
|
303
312
|
|
|
304
313
|
expect(downloadPolicy('image', 2)).toEqual({ download: true });
|
|
305
|
-
expect(downloadPolicy('image', 3)).toEqual({ download:
|
|
314
|
+
expect(downloadPolicy('image', 3)).toEqual({ download: true }); // would-blow-disk still allowed
|
|
315
|
+
expect(downloadPolicy('image', INLINE_MEDIA_MAX_BYTES)).toEqual({ download: true }); // up to the per-message cap
|
|
316
|
+
|
|
317
|
+
// ...but the per-MESSAGE size gate still rejects oversize media.
|
|
318
|
+
expect(downloadPolicy('image', INLINE_MEDIA_MAX_BYTES + 1)).toEqual({ download: false, reason: 'too_large' });
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// ── maxUserBytes / userDirBytes / enforceUserCap (per-user rolling cap) ──
|
|
322
|
+
|
|
323
|
+
// Backdate a stored item's capturedAt so the oldest-first eviction order is
|
|
324
|
+
// deterministic regardless of write timing (sub-ms writes share a clock).
|
|
325
|
+
function backdate(userId: string, messageId: string, capturedAt: number) {
|
|
326
|
+
const p = mediaPaths(userId, messageId)!;
|
|
327
|
+
const sidecar = JSON.parse(readFileSync(p.metaPath, 'utf8'));
|
|
328
|
+
sidecar.capturedAt = capturedAt;
|
|
329
|
+
writeFileSync(p.metaPath, JSON.stringify(sidecar));
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
test('maxUserBytes defaults to 1GB', () => {
|
|
333
|
+
expect(maxUserBytes()).toBe(1024 * 1024 * 1024);
|
|
334
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '2048';
|
|
335
|
+
expect(maxUserBytes()).toBe(2048);
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
test('userDirBytes counts only that user payloads, sidecars excluded', () => {
|
|
339
|
+
writeMedia(USER, 'm1', bytes('12345'), { mime: 'image/png' });
|
|
340
|
+
writeMedia(USER, 'm2', bytes('1234567890'), { mime: 'image/png' });
|
|
341
|
+
writeMedia('7', 'other', bytes('123'), { mime: 'image/png' });
|
|
342
|
+
|
|
343
|
+
expect(userDirBytes(USER)).toBe(15); // 5 + 10, sidecar JSON not counted
|
|
344
|
+
expect(userDirBytes('7')).toBe(3); // scoped per user
|
|
345
|
+
expect(userDirBytes('never')).toBe(0); // no dir yet
|
|
346
|
+
expect(userDirBytes('..')).toBe(0); // unsanitizable id
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
// Writing past 1GB (here: a tiny cap) evicts THAT user's OLDEST media, oldest
|
|
350
|
+
// first, until they fit — the new media survives. Stored under a generous cap
|
|
351
|
+
// and backdated to fix the age order, then enforced under the tight cap so the
|
|
352
|
+
// eviction is deterministic (not at the mercy of the inline write hook).
|
|
353
|
+
test('writing past the user cap evicts the user oldest media first', () => {
|
|
354
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '1000'; // generous: no inline eviction
|
|
355
|
+
writeMedia(USER, 'm1', bytes('12345'), { mime: 'image/png' }); backdate(USER, 'm1', 1); // 5
|
|
356
|
+
writeMedia(USER, 'm2', bytes('12345'), { mime: 'image/png' }); backdate(USER, 'm2', 2); // 5
|
|
357
|
+
writeMedia(USER, 'm3', bytes('12345'), { mime: 'image/png' }); backdate(USER, 'm3', 3); // 5
|
|
358
|
+
expect(mediaDiskBytes()).toBe(15);
|
|
359
|
+
|
|
360
|
+
// Tighten to 12 and enforce: 15 > 12 → evict the oldest (m1, 5) → 10 ≤ 12.
|
|
361
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '12';
|
|
362
|
+
expect(enforceUserCap(USER)).toBe(1);
|
|
363
|
+
|
|
364
|
+
expect(mediaExists(USER, 'm1')).toBeNull(); // oldest gone
|
|
365
|
+
expect(mediaExists(USER, 'm2')).not.toBeNull();
|
|
366
|
+
expect(mediaExists(USER, 'm3')).not.toBeNull();
|
|
367
|
+
expect(userDirBytes(USER)).toBe(10);
|
|
368
|
+
expect(mediaDiskBytes()).toBe(10); // global total kept honest
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
test('enforceUserCap evicts multiple oldest items when the dir blows well past the cap', () => {
|
|
372
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '1000'; // generous during the writes
|
|
373
|
+
writeMedia(USER, 'a', bytes('111'), { mime: 'image/png' }); backdate(USER, 'a', 1); // 3
|
|
374
|
+
writeMedia(USER, 'b', bytes('222'), { mime: 'image/png' }); backdate(USER, 'b', 2); // 3
|
|
375
|
+
writeMedia(USER, 'c', bytes('3333333333'), { mime: 'image/png' }); backdate(USER, 'c', 3); // 10
|
|
376
|
+
|
|
377
|
+
// 16 bytes, cap 6 → evict a (3) → 13, evict b (3) → 10, evict c (10) → 0.
|
|
378
|
+
// Even the single 10-byte file exceeds the cap, so the loop empties the dir.
|
|
379
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '6';
|
|
380
|
+
expect(enforceUserCap(USER)).toBe(3);
|
|
381
|
+
expect(userDirBytes(USER)).toBe(0);
|
|
382
|
+
expect(mediaDiskBytes()).toBe(0);
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
test('enforceUserCap evicts one user without touching another (independent caps)', () => {
|
|
386
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '1000'; // generous during the writes
|
|
387
|
+
writeMedia(USER, 'a', bytes('111'), { mime: 'image/png' }); backdate(USER, 'a', 1); // 3
|
|
388
|
+
writeMedia(USER, 'b', bytes('222'), { mime: 'image/png' }); backdate(USER, 'b', 2); // 3
|
|
389
|
+
writeMedia('7', 'x', bytes('12345'), { mime: 'image/png' }); backdate('7', 'x', 1); // 5
|
|
390
|
+
writeMedia('7', 'y', bytes('12345'), { mime: 'image/png' }); backdate('7', 'y', 2); // +5 = 10
|
|
391
|
+
|
|
392
|
+
// Cap 6: USER (6 bytes) is exactly at the cap → no eviction; user 7 (10) is over.
|
|
393
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '6';
|
|
394
|
+
expect(enforceUserCap(USER)).toBe(0);
|
|
395
|
+
expect(mediaExists(USER, 'a')).not.toBeNull();
|
|
396
|
+
expect(mediaExists(USER, 'b')).not.toBeNull();
|
|
397
|
+
|
|
398
|
+
// User 7 over cap evicts only user 7 oldest; USER untouched.
|
|
399
|
+
expect(enforceUserCap('7')).toBe(1);
|
|
400
|
+
expect(mediaExists('7', 'x')).toBeNull(); // user 7 oldest gone
|
|
401
|
+
expect(mediaExists('7', 'y')).not.toBeNull();
|
|
402
|
+
expect(userDirBytes(USER)).toBe(6); // the other user is intact
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
test('enforceUserCap is a no-op for a user under the cap, an empty dir and bad ids', () => {
|
|
406
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '100';
|
|
407
|
+
writeMedia(USER, 'm1', bytes('12345'), { mime: 'image/png' });
|
|
408
|
+
|
|
409
|
+
expect(enforceUserCap(USER)).toBe(0); // under cap
|
|
410
|
+
expect(mediaExists(USER, 'm1')).not.toBeNull();
|
|
411
|
+
expect(enforceUserCap('never-stored')).toBe(0); // no dir
|
|
412
|
+
expect(enforceUserCap('..')).toBe(0); // unsanitizable id
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
// NaN cap → enforceUserCap must still use the 1GB default, never treat NaN as
|
|
416
|
+
// "0, evict everything" or "Infinity, never evict via a broken comparison".
|
|
417
|
+
test('enforceUserCap falls back to the 1GB default on a malformed cap env', () => {
|
|
418
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = 'one gigabyte';
|
|
419
|
+
writeMedia(USER, 'm1', bytes('12345'), { mime: 'image/png' });
|
|
420
|
+
|
|
421
|
+
expect(maxUserBytes()).toBe(1024 * 1024 * 1024);
|
|
422
|
+
expect(enforceUserCap(USER)).toBe(0); // 5 bytes ≪ 1GB → nothing evicted
|
|
423
|
+
expect(mediaExists(USER, 'm1')).not.toBeNull();
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
// The post-write hook: writeMedia itself enforces the cap, so a caller that
|
|
427
|
+
// just writes (no explicit enforceUserCap) still rolls the oldest off. Here
|
|
428
|
+
// capturedAt ties on the shared clock, so the eviction falls back to file
|
|
429
|
+
// mtime ordering — the regression that proves the mtime path works.
|
|
430
|
+
test('writeMedia enforces the cap inline, falling back to mtime order on a clock tie', () => {
|
|
431
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '12';
|
|
432
|
+
const first = mediaPaths(USER, 'm1')!;
|
|
433
|
+
writeMedia(USER, 'm1', bytes('12345'), { mime: 'image/png' });
|
|
434
|
+
// Make m1 unambiguously older by mtime AND drop its sidecar capturedAt so
|
|
435
|
+
// the age source falls back to mtime.
|
|
436
|
+
const sidecar = JSON.parse(readFileSync(first.metaPath, 'utf8'));
|
|
437
|
+
delete sidecar.capturedAt;
|
|
438
|
+
writeFileSync(first.metaPath, JSON.stringify(sidecar));
|
|
439
|
+
const past = (Date.now() - 60000) / 1000;
|
|
440
|
+
utimesSync(first.dataPath, past, past);
|
|
441
|
+
|
|
442
|
+
writeMedia(USER, 'm2', bytes('12345'), { mime: 'image/png' }); // 10 ≤ 12 → no eviction
|
|
443
|
+
writeMedia(USER, 'm3', bytes('12345'), { mime: 'image/png' }); // 15 > 12 → evict oldest (m1)
|
|
444
|
+
|
|
445
|
+
expect(mediaExists(USER, 'm1')).toBeNull();
|
|
446
|
+
expect(mediaExists(USER, 'm2')).not.toBeNull();
|
|
447
|
+
expect(mediaExists(USER, 'm3')).not.toBeNull();
|
|
448
|
+
expect(userDirBytes(USER)).toBe(10);
|
|
306
449
|
});
|
|
307
450
|
|
|
308
451
|
// ── sweepExpired ──
|
|
@@ -425,13 +568,24 @@ test('resolveMediaForMessage re-checks the real byte size after download', async
|
|
|
425
568
|
expect(mediaExists(USER, MSG_ID)).toBeNull();
|
|
426
569
|
});
|
|
427
570
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
571
|
+
// The download always succeeds now; the per-user cap is honoured AFTER the
|
|
572
|
+
// write by evicting the user's oldest, so the new media is always available.
|
|
573
|
+
test('resolveMediaForMessage stores the new media and evicts the oldest under the cap', async () => {
|
|
574
|
+
process.env.WHATSAPP_MEDIA_MAX_USER_BYTES = '12';
|
|
575
|
+
// An older payload (backdated so it is unambiguously the oldest).
|
|
576
|
+
writeMedia(USER, 'oldest', bytes('12345678'), { mime: 'image/png' }); // 8 used
|
|
577
|
+
const oldPaths = mediaPaths(USER, 'oldest')!;
|
|
578
|
+
const sidecar = JSON.parse(readFileSync(oldPaths.metaPath, 'utf8'));
|
|
579
|
+
sidecar.capturedAt = 1; // ancient
|
|
580
|
+
writeFileSync(oldPaths.metaPath, JSON.stringify(sidecar));
|
|
581
|
+
|
|
582
|
+
// 10 real bytes arrive → 18 > 12 → the oldest (8) is rolled off, leaving 10.
|
|
432
583
|
const msg = mediaMsg({}, { size: undefined });
|
|
433
584
|
expect(await resolveMediaForMessage(USER, msg))
|
|
434
|
-
.toEqual({ mediaStatus: '
|
|
585
|
+
.toEqual({ mediaStatus: 'available', mediaMime: 'image/jpeg', mediaFilename: 'beach.jpg', mediaSize: 10 });
|
|
586
|
+
expect(mediaExists(USER, MSG_ID)).not.toBeNull(); // new media kept
|
|
587
|
+
expect(mediaExists(USER, 'oldest')).toBeNull(); // oldest evicted
|
|
588
|
+
expect(mediaDiskBytes()).toBe(10);
|
|
435
589
|
});
|
|
436
590
|
|
|
437
591
|
test('resolveMediaForMessage maps undefined download results to expired', async () => {
|
|
@@ -473,6 +627,18 @@ test('resolveMediaForMessage falls back to the from-timestamp message id', async
|
|
|
473
627
|
expect(mediaExists(USER, '919999000001@c.us-1717000000')).not.toBeNull();
|
|
474
628
|
});
|
|
475
629
|
|
|
630
|
+
// The fallback must mirror normalizeInbound's COUNTERPARTY keying: on fromMe
|
|
631
|
+
// the `from` is the operator's own jid, shared by every chat — keying on it
|
|
632
|
+
// stores the bytes under an id the wire never advertised (host GET 404s) and
|
|
633
|
+
// collides two same-second sends to different customers on the same path.
|
|
634
|
+
test('resolveMediaForMessage keys the fromMe fallback id on the counterparty', async () => {
|
|
635
|
+
const msg = mediaMsg({ id: undefined, fromMe: true, from: '919000000001@c.us', to: '919999000002@c.us' });
|
|
636
|
+
const verdict = await resolveMediaForMessage(USER, msg);
|
|
637
|
+
expect(verdict.mediaStatus).toBe('available');
|
|
638
|
+
expect(mediaExists(USER, '919999000002@c.us-1717000000')).not.toBeNull(); // keyed on `to`…
|
|
639
|
+
expect(mediaExists(USER, '919000000001@c.us-1717000000')).toBeNull(); // …never the operator
|
|
640
|
+
});
|
|
641
|
+
|
|
476
642
|
test('resolveMediaForMessage fills mime/filename gaps and omits a missing filename', async () => {
|
|
477
643
|
const msg = mediaMsg(
|
|
478
644
|
{ downloadMedia: async () => ({ data: Buffer.from('x').toString('base64') }) },
|