meta-sovereign 0.49.0 → 0.49.2

package/js/src/automation/outbox.js

@@ -208,6 +208,10 @@ export const guardedSend = async ({
   draft,
   send,
   settings = null,
+  // The user pressed Send on this very message. Anything else is an
+  // automated send and carries `automated: true` to the sender, which
+  // refuses WhatsApp Web for it (R-Z9).
+  userConfirmed = false,
   clock = () => new Date(),
   log = () => {},
 }) => {
@@ -247,7 +251,14 @@ export const guardedSend = async ({
     } catch (error) {
       return refused('outbox-unavailable', `not sent: ${error.message}`);
     }
-    return await deliverOnce({ store, record, draft, send, clock, log });
+    return await deliverOnce({
+      store,
+      record,
+      draft: { ...draft, automated: !userConfirmed },
+      send,
+      clock,
+      log,
+    });
   } finally {
     running.delete(id);
   }

package/js/src/broadcast/index.js

@@ -93,7 +93,13 @@ const postOn = async (name, content, outbox, resolveSource) => {
       chat: content?.chat ?? null,
       text: typeof content === 'string' ? content : (content?.text ?? ''),
     },
-    send: () => asSend(name, 'post', () => live.post(content)),
+    send: ({ automated }) =>
+      asSend(name, 'post', () =>
+        // WhatsApp refuses an automated personal send (R-Z9).
+        name === 'whatsapp'
+          ? live.post(content, { automated })
+          : live.post(content)
+      ),
   });
   return outcome.status === 'sent'
     ? { network: name, ok: true, result: outcome.result }

package/js/src/crm/reengage.js

@@ -525,7 +525,7 @@ export const createReengageSession = ({
     index += 1;
     return current();
   };
-  const sendCurrent = async () => {
+  const sendCurrent = async (userConfirmed = false) => {
     const draft = requireCurrent();
     if (typeof send !== 'function') {
       throw new Error('re-engage session has no sender; use markSent()');
@@ -536,6 +536,7 @@ export const createReengageSession = ({
       owner: 'reengage',
       draft,
       send,
+      userConfirmed,
       clock,
     });
     if (outcome.status === 'sent') {
@@ -579,7 +580,7 @@ export const createReengageSession = ({
       return items[index];
     },
     /** Send the current draft — the one action safe mode allows. */
-    confirm: sendCurrent,
+    confirm: () => sendCurrent(true),
     /** The user sent it in their own app; record it and move on. */
     async markSent() {
       const draft = requireCurrent();

package/js/src/greetings/index.js

@@ -151,8 +151,12 @@ const sendEnvelope = async (
       contact: envelope.contact,
       text: envelope.text ?? `sticker:${envelope.sticker?.id ?? ''}`,
     },
-    send: () =>
-      live.post(outgoingContent(envelope), credentials[envelope.network] ?? {}),
+    send: ({ automated }) =>
+      live.post(outgoingContent(envelope), {
+        ...(credentials[envelope.network] ?? {}),
+        // WhatsApp refuses an automated personal send (R-Z9).
+        ...(envelope.network === 'whatsapp' ? { automated } : {}),
+      }),
     clock,
   });
   if (outcome.status !== 'sent') {

package/js/src/server/index.js

@@ -87,6 +87,8 @@ import { createChatIndex } from './chat-index.js';
 import { handleChatRoutes } from './routes-chat.js';
 import { createClinkReplyStore } from '../replies/reply-store-clink.js';
 import { handleAutoImportRoutes } from './routes-auto-import.js';
+import { handleWhatsAppRoutes } from './routes-whatsapp.js';
+import { profileDirIn } from '../sources/whatsapp-web-signin.js';
 import { handleTelegramRoutes } from './routes-telegram.js';
 import { handleMessageRoutes } from './routes-messages.js';
 import { handleEmailGets } from './routes-email.js';
@@ -573,6 +575,7 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => handleEventRoutes(store, req, res, p, ctx),
     () => ctx.clientLog?.(req, res, p, ctx) ?? false,
     () => handleAutoImportRoutes(store, req, res, p, ctx),
+    () => handleWhatsAppRoutes(store, req, res, p, ctx),
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
     () => handleEmailGets(store, req, res, p, url, ctx),
@@ -721,6 +724,26 @@ const attachSockets = (server, ctx, store, { enableSync, node }) => {
   return { wsHandle, signaling };
 };
 
+// The browser sender signs in to the web apps in profiles inside the store
+// (owner only), the same ones "Sign in to WhatsApp Web" opens (R-Z9).
+const withStoreProfiles = (automations = {}, storeDir) => ({
+  ...automations,
+  browser: {
+    profileDirFor: (source) => profileDirIn(storeDir, source),
+    ...automations.browser,
+  },
+});
+
+// The open connections of `server`, so a shutdown can end them.
+const trackSockets = (server) => {
+  const sockets = new Set();
+  server.on('connection', (socket) => {
+    sockets.add(socket);
+    socket.on('close', () => sockets.delete(socket));
+  });
+  return sockets;
+};
+
 // The automations' part of the context, fed by the Telegram account.
 const automationFor = (store, automations, log, telegramAccount, later) =>
   automationContext(
@@ -783,6 +806,7 @@ export const startServer = async ({
   });
   // Services that run in the background start only after the port is ours.
   const later = [];
+  automations = withStoreProfiles(automations, storeDir);
   const chatIndex = createChatIndex(store);
   const telegramAccount = telegramAccountFor(
     store,
@@ -831,11 +855,7 @@ export const startServer = async ({
     chatIndex,
   };
   const server = createHttpServer(store, ctx, log);
-  const sockets = new Set();
-  server.on('connection', (socket) => {
-    sockets.add(socket);
-    socket.on('close', () => sockets.delete(socket));
-  });
+  const sockets = trackSockets(server);
   let bus = null;
   if (enableHandlers) {
     bus = registerDefaultHandlers(store, { browser: () => ctx.send });
@@ -882,6 +902,7 @@ export const startServer = async ({
         ctx.replies.close().catch(() => {});
         ctx.closeAutomations().catch(() => {});
         ctx.autoImport?.close();
+        ctx.whatsappSignIn?.stop().catch(() => {});
         ctx.chatIndex.close();
         stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
         wsHandle?.close();

package/js/src/server/routes-whatsapp.js

@@ -0,0 +1,103 @@
+/**
+ * WhatsApp routes (R-Z9).
+ *
+ *   GET  /api/whatsapp/web             -> { ok, state, open, signedIn, signedInAt, terms }
+ *   POST /api/whatsapp/web/sign-in     -> opens the visible WhatsApp Web window
+ *   POST /api/whatsapp/web/close       -> closes it
+ *   POST /api/whatsapp/archive {name, data(base64)}
+ *                                      -> imports an "Export chat" .txt or .zip
+ *
+ * The sign-in window is for the person to scan the QR code themselves
+ * (see ../sources/whatsapp-web-signin.js); these routes never touch it.
+ */
+
+import { json, readBody } from './util.js';
+import { isTrustedMutation } from './routes-auto-import.js';
+import { safeMessage } from './redact.js';
+import { createWhatsAppWebSignIn } from '../sources/whatsapp-web-signin.js';
+import { importWhatsAppFile } from '../sources/whatsapp-file.js';
+
+const PREFIX = '/api/whatsapp/';
+
+// What the page must tell the person (also in the docs, four languages).
+export const WHATSAPP_TERMS = Object.freeze({
+  automation: false,
+  note: "WhatsApp's terms do not allow automated or bulk use of WhatsApp Web. Meta Sovereign only types and sends a message you confirm yourself.",
+});
+
+const signInFor = (ctx) => {
+  ctx.whatsappSignIn ??= createWhatsAppWebSignIn({
+    storeDir: ctx.storeDir,
+    ...(ctx.whatsappSignInOptions ?? {}),
+    log: ctx.log,
+  });
+  return ctx.whatsappSignIn;
+};
+
+const reply = (res, status, body) =>
+  json(res, status, { ok: status < 400, ...body }) ?? true;
+
+const failure = (res, error) =>
+  reply(res, error.code === 'browser-unavailable' ? 503 : 500, {
+    error: safeMessage(error),
+    code: error.code ?? 'internal',
+  });
+
+const archiveRoute = async (store, req, res) => {
+  const body = await readBody(req, { limit: 64 * 1024 * 1024 });
+  const bytes = Buffer.from(String(body.data ?? ''), 'base64');
+  if (bytes.length === 0 || typeof body.name !== 'string') {
+    return reply(res, 400, {
+      error: 'name and data (base64) are required',
+      code: 'bad-request',
+    });
+  }
+  try {
+    return reply(res, 200, await importWhatsAppFile(store, body.name, bytes));
+  } catch (error) {
+    return reply(res, 400, { error: safeMessage(error), code: 'bad-archive' });
+  }
+};
+
+const webRoute = async (req, res, ctx, action) => {
+  const signIn = signInFor(ctx);
+  if (req.method === 'GET') {
+    return reply(res, 200, {
+      ...(await signIn.status()),
+      terms: WHATSAPP_TERMS,
+    });
+  }
+  try {
+    const status =
+      action === 'sign-in' ? await signIn.start() : await signIn.stop();
+    return reply(res, 200, { ...status, terms: WHATSAPP_TERMS });
+  } catch (error) {
+    return failure(res, error);
+  }
+};
+
+export const handleWhatsAppRoutes = async (store, req, res, p, ctx) => {
+  if (!p.startsWith(PREFIX)) {
+    return false;
+  }
+  const route = p.slice(PREFIX.length);
+  const isWeb =
+    route === 'web' || route === 'web/sign-in' || route === 'web/close';
+  const isArchive = route === 'archive';
+  if (!isWeb && !isArchive) {
+    return false;
+  }
+  if (req.method === 'POST' && !isTrustedMutation(req, ctx.allowOrigins)) {
+    return reply(res, 403, {
+      error: 'this must be started from the meta-sovereign app',
+      code: 'forbidden',
+    });
+  }
+  if (isArchive) {
+    return req.method === 'POST' ? archiveRoute(store, req, res) : false;
+  }
+  const expected = route === 'web' ? 'GET' : 'POST';
+  return req.method === expected
+    ? webRoute(req, res, ctx, route.split('/')[1])
+    : false;
+};

package/js/src/sources/auto-import/files.js

@@ -9,9 +9,9 @@ import { getSource, stampSourceLink, buildMessageLink } from '../index.js';
 import { archiveReaderFor } from './archive.js';
 import { autoImportRecipes } from './recipes.js';
 
-const toArchive = (recipe, name, text) => {
+const toArchive = (recipe, name, text, context = {}) => {
   if (recipe.toArchive) {
-    return recipe.toArchive(name, text);
+    return recipe.toArchive(name, text, context);
   }
   const trimmed = text.trim();
   if (/\.json$/i.test(name) || /^[[{]/.test(trimmed)) {
@@ -24,8 +24,8 @@ const toArchive = (recipe, name, text) => {
   return text;
 };
 
-const parseEntry = async (provider, recipe, name, data) => {
-  const archive = toArchive(recipe, name, data.toString('utf8'));
+const parseEntry = async (provider, recipe, name, data, context) => {
+  const archive = toArchive(recipe, name, data.toString('utf8'), context);
   return (await getSource(provider).parseArchive(archive)) ?? [];
 };
 
@@ -47,6 +47,7 @@ export const importDownloadedFile = async ({ store, provider, filePath }) => {
     : recipe.files.test(name)
       ? [{ name, read: () => buf }]
       : [];
+  const names = reader ? reader(buf).map((entry) => entry.name) : [];
   const reports = [];
   for (const entry of candidates) {
     const label = reader ? `${name}:${entry.name}` : name;
@@ -55,7 +56,10 @@ export const importDownloadedFile = async ({ store, provider, filePath }) => {
         provider,
         recipe,
         entry.name,
-        entry.read()
+        entry.read(),
+        // The container's own name and file list (WhatsApp names the chat
+        // after the zip and finds the media in it).
+        { container: reader ? name : null, entries: names }
       );
       for (const link of links) {
         await store.put(stampSourceLink(link, provider));

package/js/src/sources/auto-import/recipes.js

@@ -22,6 +22,7 @@
  */
 
 import { parseCsv } from './archive.js';
+import { chatNameFromFile } from '../whatsapp-ids.js';
 
 // X archives ship data as `window.YTD.<name>.part0 = [ ... ]`.
 const ytdPayload = (text) => JSON.parse(text.replace(/^[^=[]*=\s*/, ''));
@@ -97,7 +98,14 @@ export const autoImportRecipes = {
   whatsapp: {
     url: 'https://web.whatsapp.com/',
     domains: ['whatsapp.com'],
+    // An "Export chat" .txt, or the .zip of one with media (R-Z9); the
+    // chat is named after the file (or the zip holding `_chat.txt`).
     files: /\.txt$/i,
+    toArchive: (name, text, context = {}) => ({
+      text,
+      fileName: chatNameFromFile(name) ? name : (context.container ?? name),
+      files: context.entries ?? [],
+    }),
     scrape: {
       chat: '#main header span[dir="auto"]',
       message: '#main div[data-id]',

package/js/src/sources/browser-sender.js

@@ -117,9 +117,16 @@ export const extractMessages = (spec) => {
     const match = raw && pattern ? pattern.exec(raw) : null;
     return match ? match[1] : raw;
   };
+  const chatPattern = spec.chatPattern ? new RegExp(spec.chatPattern) : null;
+  const chatOf = (node) => {
+    const raw = spec.idAttribute ? node.getAttribute(spec.idAttribute) : null;
+    const match = raw && chatPattern ? chatPattern.exec(raw) : null;
+    return match ? match[1] : null;
+  };
   const rows = Array.from(document.querySelectorAll(spec.item));
   return rows.slice(-spec.limit).map((node) => ({
     id: idOf(node),
+    ...(chatPattern ? { chat: chatOf(node) } : {}),
     sender: squash(pick(node, spec.sender, spec.senderAttribute)),
     body: String(pick(node, spec.body) ?? '').trim(),
     time: squash(pick(node, spec.time, spec.timeAttribute)) || null,
@@ -169,7 +176,7 @@ const toMessageLink = (source, chat, row) => {
 };
 
 /** First candidate of a comma-separated selector list that matches. */
-const firstMatch = async (commander, selector) => {
+export const firstMatch = async (commander, selector) => {
   for (const candidate of selectorCandidates(selector)) {
     if ((await commander.count({ selector: candidate })) > 0) {
       return candidate;
@@ -488,6 +495,11 @@ export const createBrowserSender = ({
     async readRecent({ source, chat, to, limit = 20 } = {}) {
       const ui = uiFor(source);
       const target = ui.target({ chat, to });
+      // A stable chat id (E.164 number / group JID), not the typed form
+      // and not the page's title (R-Z9).
+      const normalize = (value) => ui.normalizeChat?.(value) ?? null;
+      const chatOf = (row) =>
+        normalize(row.chat) ?? normalize(target) ?? String(target);
       const rows = await inChat(source, target, {}, (commander) =>
         commander.evaluate({
           fn: extractMessages,
@@ -496,7 +508,7 @@ export const createBrowserSender = ({
       );
       return (Array.isArray(rows) ? rows : [])
         .filter((row) => row && String(row.body ?? '').length > 0)
-        .map((row) => toMessageLink(source, String(target), row));
+        .map((row) => toMessageLink(source, chatOf(row), row));
     },
 
     /** Networks with an open page. */
@@ -572,6 +584,10 @@ export const createFallbackSender =
         apiError ? { apiError: apiError.message } : {}
       );
     } catch (error) {
+      // "Not allowed to automate this" is the answer, not a pair of failures.
+      if (error?.code === 'unsupported') {
+        throw error;
+      }
       if (!apiError) {
         throw error;
       }
@@ -582,6 +598,26 @@ export const createFallbackSender =
     }
   };
 
+/**
+ * Networks whose web app may only be driven for a send the user confirms:
+ * WhatsApp's terms forbid automated and bulk use of Web, so an automated
+ * draft (`automated: true`, set by `guardedSend` for everything the user
+ * did not press Send on) is refused with `unsupported` (R-Z9).
+ */
+export const WEB_USER_CONFIRMED_ONLY = Object.freeze(['whatsapp']);
+
+const refuseAutomatedWeb = (draft) => {
+  if (
+    draft?.automated === true &&
+    WEB_USER_CONFIRMED_ONLY.includes(draft.source)
+  ) {
+    throw codedError(
+      `${draft.source}: an automated send cannot go through the web app (its terms forbid automation); send it yourself, or connect the Business Cloud API`,
+      'unsupported'
+    );
+  }
+};
+
 /**
  * The sender automations and the chat composer use: the source adapter's
  * `live.post` first, the official web UI second. It also does every other
@@ -624,13 +660,15 @@ export const createDefaultSender = ({
     : null;
   const send = createFallbackSender({
     apiSend: adapterSend,
-    browserSend: (draft) =>
-      browserFor().send({
+    browserSend: (draft) => {
+      refuseAutomatedWeb(draft);
+      return browserFor().send({
         source: draft.source,
         chat: draft.chat,
         to: draft.contact ?? draft.to,
         text: draft.text,
-      }),
+      });
+    },
     log,
   });
   const actions = createMessageActions({

package/js/src/sources/web-uis.js

@@ -33,6 +33,8 @@
  *   the message, shown on hover), then the menu items.
  */
 
+import { chatIdOf, parseChatRef } from './whatsapp-ids.js';
+
 /** VK peer ids of group chats start here (`messages.send` peer_id). */
 const VK_CHAT_PEER_OFFSET = 2000000000;
 
@@ -170,12 +172,25 @@ const whatsapp = {
   name: 'WhatsApp',
   loginUrl: 'https://web.whatsapp.com/',
   // The click-to-chat URL opens the chat and pre-fills the composer.
+  // Only a person's number opens a chat this way. A group, a linked id or
+  // a `title:` chat has no number: stripping its digits would open the
+  // WRONG person's chat, so it is refused (R-Z9).
   chatUrl: (target, { text = '' } = {}) => {
-    const phone = String(target).replace(/\D/g, '');
+    const ref = parseChatRef(target);
+    if (ref?.kind !== 'person') {
+      throw Object.assign(
+        new Error(
+          `whatsapp: "${target}" is not a phone number; WhatsApp Web can only be opened by number from here`
+        ),
+        { code: 'unsupported-target' }
+      );
+    }
     const query = text ? `&text=${encodeURIComponent(text)}` : '';
-    return `https://web.whatsapp.com/send?phone=${phone}${query}`;
+    return `https://web.whatsapp.com/send?phone=${ref.id.slice(1)}${query}`;
   },
   target: ({ chat, to }) => to ?? chat,
+  // The stable chat id of a JID or number (E.164 / group JID).
+  normalizeChat: chatIdOf,
   loggedIn: '#pane-side, div[aria-label="Chat list"], #main footer',
   loggedOut: 'canvas[aria-label="Scan me!"], div[data-ref], .landing-main',
   composer:
@@ -191,7 +206,9 @@ const whatsapp = {
     idAttribute: 'data-id',
     // `data-id` is `<out>_<chat jid>_<message id>`: the message id is
     // what the Cloud API calls `wamid`'s last part.
-    idPattern: '_([^_]+)$',
+    // Groups add the participant: `false_<group>@g.us_<msg>_<who>@c.us`.
+    idPattern: '^(?:true|false)_[^_]+_([^_]+)',
+    chatPattern: '^(?:true|false)_([^_]+)_',
     sender: '[data-pre-plain-text]',
     senderAttribute: 'data-pre-plain-text',
     body: 'span.selectable-text, .copyable-text span',