meta-sovereign 0.49.0 → 0.49.3

package/js/src/automation/outbox.js

@@ -208,6 +208,10 @@ export const guardedSend = async ({
   draft,
   send,
   settings = null,
+  // The user pressed Send on this very message. Anything else is an
+  // automated send and carries `automated: true` to the sender, which
+  // refuses WhatsApp Web for it (R-Z9).
+  userConfirmed = false,
   clock = () => new Date(),
   log = () => {},
 }) => {
@@ -247,7 +251,14 @@ export const guardedSend = async ({
     } catch (error) {
       return refused('outbox-unavailable', `not sent: ${error.message}`);
     }
-    return await deliverOnce({ store, record, draft, send, clock, log });
+    return await deliverOnce({
+      store,
+      record,
+      draft: { ...draft, automated: !userConfirmed },
+      send,
+      clock,
+      log,
+    });
   } finally {
     running.delete(id);
   }

package/js/src/broadcast/index.js

@@ -93,7 +93,13 @@ const postOn = async (name, content, outbox, resolveSource) => {
       chat: content?.chat ?? null,
       text: typeof content === 'string' ? content : (content?.text ?? ''),
     },
-    send: () => asSend(name, 'post', () => live.post(content)),
+    send: ({ automated }) =>
+      asSend(name, 'post', () =>
+        // WhatsApp refuses an automated personal send (R-Z9).
+        name === 'whatsapp'
+          ? live.post(content, { automated })
+          : live.post(content)
+      ),
   });
   return outcome.status === 'sent'
     ? { network: name, ok: true, result: outcome.result }

package/js/src/crm/reengage.js

@@ -525,7 +525,7 @@ export const createReengageSession = ({
     index += 1;
     return current();
   };
-  const sendCurrent = async () => {
+  const sendCurrent = async (userConfirmed = false) => {
     const draft = requireCurrent();
     if (typeof send !== 'function') {
       throw new Error('re-engage session has no sender; use markSent()');
@@ -536,6 +536,7 @@ export const createReengageSession = ({
       owner: 'reengage',
       draft,
       send,
+      userConfirmed,
       clock,
     });
     if (outcome.status === 'sent') {
@@ -579,7 +580,7 @@ export const createReengageSession = ({
       return items[index];
     },
     /** Send the current draft — the one action safe mode allows. */
-    confirm: sendCurrent,
+    confirm: () => sendCurrent(true),
     /** The user sent it in their own app; record it and move on. */
     async markSent() {
       const draft = requireCurrent();

package/js/src/greetings/index.js

@@ -151,8 +151,12 @@ const sendEnvelope = async (
       contact: envelope.contact,
       text: envelope.text ?? `sticker:${envelope.sticker?.id ?? ''}`,
     },
-    send: () =>
-      live.post(outgoingContent(envelope), credentials[envelope.network] ?? {}),
+    send: ({ automated }) =>
+      live.post(outgoingContent(envelope), {
+        ...(credentials[envelope.network] ?? {}),
+        // WhatsApp refuses an automated personal send (R-Z9).
+        ...(envelope.network === 'whatsapp' ? { automated } : {}),
+      }),
     clock,
   });
   if (outcome.status !== 'sent') {

package/js/src/server/aggregate.js

@@ -27,13 +27,27 @@ const blank = (id) => ({
   open: null,
 });
 
+// A message time as an ISO string. VK and WhatsApp keep unix seconds (a
+// number), which used to break the contact sort and `/api/status` (R-Z8).
+const seenAt = (value) => {
+  if (value === null || value === undefined || value === '') {
+    return null;
+  }
+  const n = Number(value);
+  if (Number.isFinite(n) && !/[-:T]/.test(String(value))) {
+    return new Date(n < 1e12 ? n * 1000 : n).toISOString();
+  }
+  return String(value);
+};
+
 const addMessage = (entry, m) => {
   entry.networks.add(m.source);
   entry.chats.add(m.chat);
   entry.messageCount += 1;
-  const latest = !entry.lastSeen || (m.timestamp ?? '') > entry.lastSeen;
+  const at = seenAt(m.timestamp);
+  const latest = !entry.lastSeen || (at ?? '') > entry.lastSeen;
   if (latest) {
-    entry.lastSeen = m.timestamp ?? null;
+    entry.lastSeen = at;
     entry.name = m.senderName ?? entry.name;
   } else {
     entry.name ??= m.senderName ?? null;

package/js/src/server/index.js

@@ -87,6 +87,8 @@ import { createChatIndex } from './chat-index.js';
 import { handleChatRoutes } from './routes-chat.js';
 import { createClinkReplyStore } from '../replies/reply-store-clink.js';
 import { handleAutoImportRoutes } from './routes-auto-import.js';
+import { handleWhatsAppRoutes } from './routes-whatsapp.js';
+import { profileDirIn } from '../sources/whatsapp-web-signin.js';
 import { handleTelegramRoutes } from './routes-telegram.js';
 import { handleMessageRoutes } from './routes-messages.js';
 import { handleEmailGets } from './routes-email.js';
@@ -102,6 +104,7 @@ import { createGitHubCheck } from '../sources/github-check.js';
 import { handleMetrics } from './metrics.js';
 import { jsonLog } from './log.js';
 import { listenLocal } from './local-tls.js';
+import { handlePullRoutes, pullContext } from './routes-pull.js';
 import { createPeer, attachSyncWebSocket } from '../sync/index.js';
 import { attachSignaling } from '../sync/webrtc-signaling.js';
 import { registerDefaultHandlers } from './handlers-bootstrap.js';
@@ -146,7 +149,9 @@ export const CSP =
   "default-src 'self'; " +
   "script-src 'self' 'wasm-unsafe-eval'; " +
   "style-src 'self'; " +
-  "img-src 'self' data: blob: https://vk.com; " +
+  "img-src 'self' data: blob: https://vk.com " +
+  // VK's picture CDNs, for photos and stickers in VK chats (R-Z8).
+  'https://*.userapi.com https://*.vk-cdn.net https://*.vkuserphoto.ru; ' +
   'frame-src https:; ' +
   "connect-src 'self' https: ws: wss: http://127.0.0.1:* http://localhost:*; " +
   "worker-src 'self'; " +
@@ -573,6 +578,8 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => handleEventRoutes(store, req, res, p, ctx),
     () => ctx.clientLog?.(req, res, p, ctx) ?? false,
     () => handleAutoImportRoutes(store, req, res, p, ctx),
+    () => handleWhatsAppRoutes(store, req, res, p, ctx),
+    () => handlePullRoutes(store, req, res, p, ctx),
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
     () => handleEmailGets(store, req, res, p, url, ctx),
@@ -721,6 +728,26 @@ const attachSockets = (server, ctx, store, { enableSync, node }) => {
   return { wsHandle, signaling };
 };
 
+// The browser sender signs in to the web apps in profiles inside the store
+// (owner only), the same ones "Sign in to WhatsApp Web" opens (R-Z9).
+const withStoreProfiles = (automations = {}, storeDir) => ({
+  ...automations,
+  browser: {
+    profileDirFor: (source) => profileDirIn(storeDir, source),
+    ...automations.browser,
+  },
+});
+
+// The open connections of `server`, so a shutdown can end them.
+const trackSockets = (server) => {
+  const sockets = new Set();
+  server.on('connection', (socket) => {
+    sockets.add(socket);
+    socket.on('close', () => sockets.delete(socket));
+  });
+  return sockets;
+};
+
 // The automations' part of the context, fed by the Telegram account.
 const automationFor = (store, automations, log, telegramAccount, later) =>
   automationContext(
@@ -764,15 +791,14 @@ export const startServer = async ({
   telegramExtraBuild,
   // Tests and demos pass a fake, so GitHub is never asked (R-Y53).
   githubCheck,
-  // The hosted app and whether this server also serves its own copy
-  // (./hosted-ui.js, R-Y58 and R-Y59).
+  // The hosted app, and whether this serves its own copy (./hosted-ui.js).
   uiUrl,
   localUi,
-  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); off
-  // unless asked for, `serve` turns it on.
+  // HTTPS on the same port for Safari (./local-tls.js, R-Y64); `serve` sets it.
   tls,
   // Tests: a fake `(dir) => bytes|null` for the free disk space (R-Z14).
   diskFree,
+  pulls,
 } = {}) => {
   const { store, resolvedArchiveDir, flush, storeHealth } = await initStore({
     providedStore,
@@ -783,6 +809,7 @@ export const startServer = async ({
   });
   // Services that run in the background start only after the port is ours.
   const later = [];
+  automations = withStoreProfiles(automations, storeDir);
   const chatIndex = createChatIndex(store);
   const telegramAccount = telegramAccountFor(
     store,
@@ -829,17 +856,13 @@ export const startServer = async ({
       later
     ),
     chatIndex,
+    ...pullContext(store, log, pulls, later),
   };
   const server = createHttpServer(store, ctx, log);
-  const sockets = new Set();
-  server.on('connection', (socket) => {
-    sockets.add(socket);
-    socket.on('close', () => sockets.delete(socket));
-  });
-  let bus = null;
-  if (enableHandlers) {
-    bus = registerDefaultHandlers(store, { browser: () => ctx.send });
-  }
+  const sockets = trackSockets(server);
+  const bus = enableHandlers
+    ? registerDefaultHandlers(store, { browser: () => ctx.send })
+    : null;
   const { wsHandle, signaling } = attachSockets(server, ctx, store, {
     enableSync,
     node,
@@ -882,6 +905,8 @@ export const startServer = async ({
         ctx.replies.close().catch(() => {});
         ctx.closeAutomations().catch(() => {});
         ctx.autoImport?.close();
+        ctx.whatsappSignIn?.stop().catch(() => {});
+        ctx.stopPulls?.();
         ctx.chatIndex.close();
         stopAll(telegramAccount, ctx.telegramAccounts, ctx.telegramExtras);
         wsHandle?.close();

package/js/src/server/routes-pull.js

@@ -0,0 +1,147 @@
+/**
+ * Inbox pulls for the networks that have no live connection of their own
+ * (R-Z8): the Connections card's "Fetch now" button and the background
+ * schedule behind it.
+ *
+ * Its routes:
+ *   POST /api/pull {source}   -> { ok, source, imported, lastOkAt }
+ *                                 or { ok: false, error, code } with `code` one of
+ *                                 unsupported | not-connected | rate-limited
+ *                                 (HTTP 429, at most one fetch per 15 s per
+ *                                 network, `retryAfterMs`) | the network's own code
+ *   GET  /api/pull/status     -> { ok, sources: { <source>: { connected,
+ *                                 lastOkAt, lastError } } }
+ *
+ * The same `createPullRunner` (../sources/pull-runner.js) serves every
+ * network: add a job to `pullJobs` to give another network a button and a
+ * schedule. VK also keeps a Long Poll running (../sources/vk-sync.js).
+ */
+
+import { createPullRunner, loadPullState } from '../sources/pull-runner.js';
+import { createVkInbox } from '../sources/vk-sync.js';
+import { liveOptionsFrom, pullLiveInto } from '../sources/index.js';
+import { createUpworkLive } from '../sources/upwork.js';
+import { createPeoplePerHourLive } from '../sources/peopleperhour.js';
+import { fail, json, readBody } from './util.js';
+
+const MINUTE = 60_000;
+
+const connected = (store, source) => async () =>
+  Boolean((await liveOptionsFrom(store, source)).token);
+
+// `live` for a network whose test double or host is given; the saved token
+// still comes from the store.
+const liveOf = (make, given) => (given ? make(given) : undefined);
+
+/** The jobs: network -> { ready, run, intervalMs }. */
+export const pullJobs = (store, { vkInbox, options = {} }) => ({
+  vk: {
+    ready: vkInbox.ready,
+    run: vkInbox.fetch,
+    intervalMs: 10 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  upwork: {
+    ready: connected(store, 'upwork'),
+    run: () =>
+      pullLiveInto(store, 'upwork', {
+        live: liveOf(createUpworkLive, options.upwork),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+  peopleperhour: {
+    ready: connected(store, 'peopleperhour'),
+    run: () =>
+      pullLiveInto(store, 'peopleperhour', {
+        live: liveOf(createPeoplePerHourLive, options.peopleperhour),
+      }),
+    intervalMs: 5 * MINUTE,
+    firstDelayMs: 30_000,
+  },
+});
+
+/**
+ * The pull services for a server: `{ pullRunner, vkInbox, startAll, stopAll }`.
+ * `options` (tests): `{ vk: {baseUrl, fetchImpl, wait}, upwork: {baseUrl},
+ * peopleperhour: {baseUrl}, minGapMs, intervals }`; `false` turns it all off.
+ */
+export const createPullServices = (store, log, options = {}) => {
+  const vkInbox = createVkInbox({ store, log, ...options.vk });
+  const jobs = pullJobs(store, { vkInbox, options });
+  for (const [source, intervalMs] of Object.entries(options.intervals ?? {})) {
+    jobs[source].intervalMs = intervalMs;
+    jobs[source].firstDelayMs = intervalMs;
+  }
+  const pullRunner = createPullRunner({
+    store,
+    jobs,
+    log,
+    minGapMs: options.minGapMs,
+  });
+  return {
+    pullRunner,
+    vkInbox,
+    start: () => {
+      pullRunner.start();
+      vkInbox.start();
+    },
+    stop: () => {
+      pullRunner.stop();
+      vkInbox.stop();
+    },
+  };
+};
+
+const statusOf = async (store, ctx) => {
+  const sources = {};
+  for (const source of ctx.pullRunner.sources()) {
+    const state = await loadPullState(store, source);
+    sources[source] = {
+      connected: await ctx.pullRunner.connected(source),
+      lastOkAt: state.lastOkAt ?? null,
+      lastError: state.lastError ?? null,
+    };
+  }
+  return { ok: true, sources };
+};
+
+const sourceOf = (body) =>
+  typeof body.source === 'string' && /^[a-z]{2,30}$/.test(body.source)
+    ? body.source
+    : null;
+
+export const handlePullRoutes = async (store, req, res, p, ctx) => {
+  if (!ctx.pullRunner) {
+    return false;
+  }
+  if (p === '/api/pull/status' && req.method === 'GET') {
+    return json(res, 200, await statusOf(store, ctx)) ?? true;
+  }
+  if (p !== '/api/pull' || req.method !== 'POST') {
+    return false;
+  }
+  const source = sourceOf(await readBody(req));
+  if (!source) {
+    return fail(res, 400, 'bad-request', 'a "source" is needed') ?? true;
+  }
+  const outcome = await ctx.pullRunner.fetchNow(source);
+  if (outcome.ok) {
+    return json(res, 200, outcome) ?? true;
+  }
+  const status =
+    { 'rate-limited': 429, unsupported: 404, 'not-connected': 409 }[
+      outcome.code
+    ] ?? 502;
+  return json(res, status, outcome) ?? true;
+};
+
+/** What a server keeps for the pulls; they start with the other `later` services. */
+export const pullContext = (store, log, options, later) => {
+  if (options === false) {
+    return { pullRunner: null };
+  }
+  const services = createPullServices(store, log, options ?? {});
+  later.push(services.start);
+  return { pullRunner: services.pullRunner, stopPulls: services.stop };
+};

package/js/src/server/routes-whatsapp.js

@@ -0,0 +1,103 @@
+/**
+ * WhatsApp routes (R-Z9).
+ *
+ *   GET  /api/whatsapp/web             -> { ok, state, open, signedIn, signedInAt, terms }
+ *   POST /api/whatsapp/web/sign-in     -> opens the visible WhatsApp Web window
+ *   POST /api/whatsapp/web/close       -> closes it
+ *   POST /api/whatsapp/archive {name, data(base64)}
+ *                                      -> imports an "Export chat" .txt or .zip
+ *
+ * The sign-in window is for the person to scan the QR code themselves
+ * (see ../sources/whatsapp-web-signin.js); these routes never touch it.
+ */
+
+import { json, readBody } from './util.js';
+import { isTrustedMutation } from './routes-auto-import.js';
+import { safeMessage } from './redact.js';
+import { createWhatsAppWebSignIn } from '../sources/whatsapp-web-signin.js';
+import { importWhatsAppFile } from '../sources/whatsapp-file.js';
+
+const PREFIX = '/api/whatsapp/';
+
+// What the page must tell the person (also in the docs, four languages).
+export const WHATSAPP_TERMS = Object.freeze({
+  automation: false,
+  note: "WhatsApp's terms do not allow automated or bulk use of WhatsApp Web. Meta Sovereign only types and sends a message you confirm yourself.",
+});
+
+const signInFor = (ctx) => {
+  ctx.whatsappSignIn ??= createWhatsAppWebSignIn({
+    storeDir: ctx.storeDir,
+    ...(ctx.whatsappSignInOptions ?? {}),
+    log: ctx.log,
+  });
+  return ctx.whatsappSignIn;
+};
+
+const reply = (res, status, body) =>
+  json(res, status, { ok: status < 400, ...body }) ?? true;
+
+const failure = (res, error) =>
+  reply(res, error.code === 'browser-unavailable' ? 503 : 500, {
+    error: safeMessage(error),
+    code: error.code ?? 'internal',
+  });
+
+const archiveRoute = async (store, req, res) => {
+  const body = await readBody(req, { limit: 64 * 1024 * 1024 });
+  const bytes = Buffer.from(String(body.data ?? ''), 'base64');
+  if (bytes.length === 0 || typeof body.name !== 'string') {
+    return reply(res, 400, {
+      error: 'name and data (base64) are required',
+      code: 'bad-request',
+    });
+  }
+  try {
+    return reply(res, 200, await importWhatsAppFile(store, body.name, bytes));
+  } catch (error) {
+    return reply(res, 400, { error: safeMessage(error), code: 'bad-archive' });
+  }
+};
+
+const webRoute = async (req, res, ctx, action) => {
+  const signIn = signInFor(ctx);
+  if (req.method === 'GET') {
+    return reply(res, 200, {
+      ...(await signIn.status()),
+      terms: WHATSAPP_TERMS,
+    });
+  }
+  try {
+    const status =
+      action === 'sign-in' ? await signIn.start() : await signIn.stop();
+    return reply(res, 200, { ...status, terms: WHATSAPP_TERMS });
+  } catch (error) {
+    return failure(res, error);
+  }
+};
+
+export const handleWhatsAppRoutes = async (store, req, res, p, ctx) => {
+  if (!p.startsWith(PREFIX)) {
+    return false;
+  }
+  const route = p.slice(PREFIX.length);
+  const isWeb =
+    route === 'web' || route === 'web/sign-in' || route === 'web/close';
+  const isArchive = route === 'archive';
+  if (!isWeb && !isArchive) {
+    return false;
+  }
+  if (req.method === 'POST' && !isTrustedMutation(req, ctx.allowOrigins)) {
+    return reply(res, 403, {
+      error: 'this must be started from the meta-sovereign app',
+      code: 'forbidden',
+    });
+  }
+  if (isArchive) {
+    return req.method === 'POST' ? archiveRoute(store, req, res) : false;
+  }
+  const expected = route === 'web' ? 'GET' : 'POST';
+  return req.method === expected
+    ? webRoute(req, res, ctx, route.split('/')[1])
+    : false;
+};

package/js/src/sources/auto-import/files.js

@@ -9,9 +9,9 @@ import { getSource, stampSourceLink, buildMessageLink } from '../index.js';
 import { archiveReaderFor } from './archive.js';
 import { autoImportRecipes } from './recipes.js';
 
-const toArchive = (recipe, name, text) => {
+const toArchive = (recipe, name, text, context = {}) => {
   if (recipe.toArchive) {
-    return recipe.toArchive(name, text);
+    return recipe.toArchive(name, text, context);
   }
   const trimmed = text.trim();
   if (/\.json$/i.test(name) || /^[[{]/.test(trimmed)) {
@@ -24,8 +24,12 @@ const toArchive = (recipe, name, text) => {
   return text;
 };
 
-const parseEntry = async (provider, recipe, name, data) => {
-  const archive = toArchive(recipe, name, data.toString('utf8'));
+const parseEntry = async (provider, recipe, name, data, context) => {
+  // A recipe may decode its own bytes (VK's Windows-1251 pages, R-Z8).
+  const text = recipe.decode
+    ? recipe.decode(name, data)
+    : data.toString('utf8');
+  const archive = toArchive(recipe, name, text, context);
   return (await getSource(provider).parseArchive(archive)) ?? [];
 };
 
@@ -47,6 +51,7 @@ export const importDownloadedFile = async ({ store, provider, filePath }) => {
     : recipe.files.test(name)
       ? [{ name, read: () => buf }]
       : [];
+  const names = reader ? reader(buf).map((entry) => entry.name) : [];
   const reports = [];
   for (const entry of candidates) {
     const label = reader ? `${name}:${entry.name}` : name;
@@ -55,7 +60,10 @@ export const importDownloadedFile = async ({ store, provider, filePath }) => {
         provider,
         recipe,
         entry.name,
-        entry.read()
+        entry.read(),
+        // The container's own name and file list (WhatsApp names the chat
+        // after the zip and finds the media in it).
+        { container: reader ? name : null, entries: names }
       );
       for (const link of links) {
         await store.put(stampSourceLink(link, provider));

package/js/src/sources/auto-import/recipes.js

@@ -22,6 +22,12 @@
  */
 
 import { parseCsv } from './archive.js';
+import { chatNameFromFile } from '../whatsapp-ids.js';
+import {
+  decodeVkHtml,
+  parseVkHtmlPage,
+  peerFromArchivePath,
+} from '../vk-html-archive.js';
 
 // X archives ship data as `window.YTD.<name>.part0 = [ ... ]`.
 const ytdPayload = (text) => JSON.parse(text.replace(/^[^=[]*=\s*/, ''));
@@ -63,6 +69,14 @@ const linkedinToArchive = (_name, text) => ({
   ),
 });
 
+const vkDecode = (name, bytes) =>
+  /\.html$/i.test(name) ? decodeVkHtml(bytes) : bytes.toString('utf8');
+
+const vkToArchive = (name, text) =>
+  /\.html$/i.test(name)
+    ? parseVkHtmlPage(text, { peer: peerFromArchivePath(name) })
+    : JSON.parse(text);
+
 export const autoImportRecipes = {
   email: {
     url: 'https://takeout.google.com/settings/takeout/custom/gmail',
@@ -86,7 +100,11 @@ export const autoImportRecipes = {
   vk: {
     url: 'https://vk.com/data_protection?section=rules',
     domains: ['vk.com', 'vk.ru'],
-    files: /messages.*\.json$/i,
+    // The JSON of konard/vk-export, or the pages of VK's own archive:
+    // messages/<peer>/messages<N>.html in Windows-1251 (R-Z8).
+    files: /(messages.*\.json|messages\/[^/]+\/messages\d*\.html)$/i,
+    decode: vkDecode,
+    toArchive: vkToArchive,
   },
   x: {
     url: 'https://x.com/settings/download_your_data',
@@ -97,7 +115,14 @@ export const autoImportRecipes = {
   whatsapp: {
     url: 'https://web.whatsapp.com/',
     domains: ['whatsapp.com'],
+    // An "Export chat" .txt, or the .zip of one with media (R-Z9); the
+    // chat is named after the file (or the zip holding `_chat.txt`).
     files: /\.txt$/i,
+    toArchive: (name, text, context = {}) => ({
+      text,
+      fileName: chatNameFromFile(name) ? name : (context.container ?? name),
+      files: context.entries ?? [],
+    }),
     scrape: {
       chat: '#main header span[dir="auto"]',
       message: '#main div[data-id]',