meta-sovereign 0.49.16 → 0.49.17

package/js/src/server/background-jobs.js

@@ -78,7 +78,8 @@ export const backgroundJobs = ({ ctx, store, log, later, backups }) => {
   });
   const purge = cachePurgeFor({ store, log, later });
   return {
-    // Returns a promise settled once a backup in flight has finished.
+    // Returns a promise settled once a backup and the pulls in flight
+    // have finished.
     stopJobs: () => {
       const backupDone = backup.stopBackups();
       audience.stopAudienceSync();
@@ -86,8 +87,8 @@ export const backgroundJobs = ({ ctx, store, log, later, backups }) => {
       // The other timers and pollers the server started with the port.
       ctx.autoImport?.close();
       ctx.whatsappSignIn?.stop().catch(() => {});
-      ctx.stopPulls?.();
-      return backupDone;
+      const pullsDone = ctx.stopPulls?.();
+      return Promise.all([backupDone, pullsDone]).then(() => {});
     },
   };
 };

package/js/src/server/index.js

@@ -40,6 +40,7 @@
  *   POST /api/email/send           -> local-server email send proxy
  *   GET  /api/email/attachment     -> a mail attachment from the media folder (routes-email.js)
  *   GET  /api/media/<network>/<sha256>[.ext] -> imported archive media (routes-media.js, R-Z15)
+ *   POST /api/archive/upload?source=&name= -> import an export file (routes-archive-upload.js)
  *   GET  /api/backups              -> list archives (gated on archiveDir)
  *   POST /api/backups              -> { passphrase?, keep? } -> { file }
  *   POST /api/backups/restore      -> { file, passphrase? } -> { restored }
@@ -97,6 +98,10 @@ import { handleTelegramRoutes } from './routes-telegram.js';
 import { handleMessageRoutes } from './routes-messages.js';
 import { handleEmailGets } from './routes-email.js';
 import { handleMediaGets } from './routes-media.js';
+import {
+  handleArchiveUpload,
+  UPLOAD_TIMEOUT_MS,
+} from './routes-archive-upload.js';
 import { createAttachmentStore } from './email-attachments.js';
 import {
   VK_CAPTCHA_HOSTS,
@@ -586,6 +591,7 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => ctx.clientLog?.(req, res, p, ctx) ?? false,
     () => handleAutoImportRoutes(store, req, res, p, ctx),
     () => handleWhatsAppRoutes(store, req, res, p, ctx),
+    () => handleArchiveUpload(store, req, res, p, ctx),
     () => handlePullRoutes(store, req, res, p, ctx),
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
@@ -659,7 +665,9 @@ const initStore = async ({
 // The HTTP server: each request is logged, and routed (./account-scope.js
 // picks the Telegram account it is for).
 const createHttpServer = (store, ctx, log) =>
-  http.createServer((req, res) => {
+  // A big export uploaded over Wi-Fi takes longer than Node's default 5
+  // minutes to arrive (R-Z23); headers still time out at Node's 60 s.
+  http.createServer({ requestTimeout: UPLOAD_TIMEOUT_MS }, (req, res) => {
     const start = Date.now();
     res.on('finish', () => {
       log?.({

package/js/src/server/routes-archive-upload.js

@@ -0,0 +1,220 @@
+/**
+ * Archive uploads (R-Z23):
+ *   POST /api/archive/upload?source=<network>&name=<file name>
+ *        (application/octet-stream body: the file's bytes)
+ *     -> { ok, imported, files: [{ name, imported, error? }], media?, chat? }
+ *
+ * The page's import box sends an export it was given (a Facebook, X,
+ * LinkedIn or WhatsApp .zip, a Telegram result.json, ...) here. The bytes
+ * are streamed to a private temp folder in the store (never held in
+ * memory, never base64, no fixed size cap) and read by the same importer
+ * as auto-import (../sources/auto-import/files.js: a zip is opened in
+ * place, media copied to `<store>/media/<network>/`); a WhatsApp export
+ * keeps its own importer (../sources/whatsapp-file.js), which copies the
+ * attachments its chat names. The temp file is removed afterwards, and
+ * leftovers of a crash are swept on the next upload.
+ *
+ * Not JSON, so not isTrustedMutation's check: its own, as for the Telegram
+ * upload — the bytes' type is not one a form on another site can send
+ * without asking first (CORS), and another site's page is refused. A disk
+ * without room for the file twice over (the file and what it becomes)
+ * is refused before anything is written (507).
+ */
+
+import { createWriteStream, promises as fs } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { json } from './util.js';
+import { classifyOrigin } from './cors.js';
+import { safeMessage } from './redact.js';
+import { enoughSpace } from '../storage/disk-space.js';
+import { ensurePrivateDir } from '../storage/durable.js';
+import { autoImportRecipes } from '../sources/auto-import/recipes.js';
+import { importDownloadedFile } from '../sources/auto-import/files.js';
+import { importWhatsAppPath } from '../sources/whatsapp-file.js';
+import { isFatalWrite } from '../sources/archive/context.js';
+
+export const ARCHIVE_UPLOAD_ROUTE = '/api/archive/upload';
+const UPLOADS = 'uploads';
+// A temp folder this old is a crashed upload's: no request lives that long.
+const STALE_MS = 6 * 60 * 60_000;
+/** How long a request may take to arrive (the server's `requestTimeout`). */
+export const UPLOAD_TIMEOUT_MS = 3 * 60 * 60_000;
+
+const reply = (res, status, body) =>
+  json(res, status, { ok: status < 400, ...body }) ?? true;
+
+// Refused before the body was read: drain it so the socket stays usable.
+const refuse = (req, res, status, code, error) => {
+  req.resume();
+  return reply(res, status, { error, code });
+};
+
+/** A plain file name: no folders or control characters, at most 200 long. */
+export const uploadName = (raw) => {
+  const base = path.posix.basename(String(raw ?? '').replace(/\\/g, '/'));
+  const clean = [...base]
+    .filter((ch) => ch.charCodeAt(0) >= 0x20 && ch.charCodeAt(0) !== 0x7f)
+    .join('')
+    .slice(-200);
+  return clean && clean !== '.' && clean !== '..' ? clean : 'archive';
+};
+
+const sweepStale = async (root, now = Date.now()) => {
+  for (const name of await fs.readdir(root).catch(() => [])) {
+    const dir = path.join(root, name);
+    const stat = await fs.stat(dir).catch(() => null);
+    if (stat?.isDirectory() && now - stat.mtimeMs > STALE_MS) {
+      await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
+    }
+  }
+};
+
+const importUploaded = async (store, { source, name, file, mediaDir }) => {
+  if (source === 'whatsapp') {
+    const out = await importWhatsAppPath(store, name, file, { mediaDir });
+    return { ...out, files: [{ name, imported: out.imported }] };
+  }
+  const files = await importDownloadedFile({
+    store,
+    provider: source,
+    filePath: file,
+    mediaDir,
+  });
+  return {
+    imported: files.reduce((sum, row) => sum + (row.imported ?? 0), 0),
+    files,
+  };
+};
+
+// The answer's status: a full disk, a file with nothing to import, or done.
+const statusOf = (out) => {
+  if (out.files.some((row) => row.aborted)) {
+    return 507;
+  }
+  return out.imported === 0 && out.files.every((row) => row.error) ? 400 : 200;
+};
+
+const failed = (out) => {
+  const status = statusOf(out);
+  if (status === 200) {
+    return {};
+  }
+  const row = out.files.find((r) => r.error);
+  return {
+    error: row?.error ?? 'nothing to import',
+    code: status === 507 ? 'store-write-failed' : 'bad-archive',
+  };
+};
+
+const NOT_WHOLE = Object.freeze([
+  400,
+  { error: 'the file did not arrive whole; try again', code: 'upload-failed' },
+]);
+
+// The bytes to `file`, then the import: `[status, body]`.
+const importInto = async (req, ctx, file, { source, name, size }) => {
+  try {
+    await pipeline(req, createWriteStream(file, { flags: 'wx', mode: 0o600 }));
+  } catch {
+    return NOT_WHOLE;
+  }
+  if ((await fs.stat(file)).size !== size) {
+    return NOT_WHOLE;
+  }
+  try {
+    const out = await importUploaded(ctx.store, {
+      source,
+      name,
+      file,
+      mediaDir: path.join(ctx.storeDir, 'media'),
+    });
+    return [statusOf(out), { ...out, ...failed(out) }];
+  } catch (error) {
+    // A full or unwritable disk is not a bad file (R-Z14).
+    return isFatalWrite(error)
+      ? [507, { error: safeMessage(error), code: 'store-write-failed' }]
+      : [400, { error: safeMessage(error), code: 'bad-archive' }];
+  }
+};
+
+// The temp folder is gone before the answer is sent.
+const receive = async (req, res, ctx, wanted) => {
+  const root = path.join(ctx.storeDir, UPLOADS);
+  await ensurePrivateDir(root);
+  await sweepStale(root);
+  const dir = path.join(root, randomBytes(8).toString('hex'));
+  await ensurePrivateDir(dir);
+  let answer;
+  try {
+    answer = await importInto(req, ctx, path.join(dir, wanted.name), wanted);
+  } finally {
+    await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
+  }
+  return reply(res, ...answer);
+};
+
+/** POST /api/archive/upload (see the header). */
+export const handleArchiveUpload = async (store, req, res, p, ctx) => {
+  if (p !== ARCHIVE_UPLOAD_ROUTE || req.method !== 'POST') {
+    return false;
+  }
+  const type = String(req.headers['content-type'] ?? '').toLowerCase();
+  if (
+    !type.startsWith('application/octet-stream') ||
+    classifyOrigin(req, ctx?.allowOrigins ?? []) === 'denied'
+  ) {
+    return refuse(
+      req,
+      res,
+      403,
+      'forbidden',
+      'archives are sent from the app only'
+    );
+  }
+  const query = new URL(req.url, 'http://local').searchParams;
+  const source = query.get('source') ?? '';
+  if (!Object.hasOwn(autoImportRecipes, source)) {
+    return refuse(
+      req,
+      res,
+      400,
+      'bad-source',
+      `no archive import for "${source}"`
+    );
+  }
+  if (!ctx?.storeDir) {
+    return refuse(
+      req,
+      res,
+      501,
+      'no-store',
+      'archive uploads need a store folder'
+    );
+  }
+  const size = Number(req.headers['content-length']);
+  if (!Number.isSafeInteger(size) || size <= 0) {
+    return refuse(req, res, 411, 'length-required', 'the file size is needed');
+  }
+  const space = await enoughSpace(ctx.storeDir, size * 2, ctx.diskFree);
+  if (!space.ok) {
+    return refuse(
+      req,
+      res,
+      507,
+      'low-disk',
+      `not enough free disk space for this file: about ${Math.ceil(space.needed / 1048576)} MB needed`
+    );
+  }
+  return receive(
+    req,
+    res,
+    { ...ctx, store },
+    {
+      source,
+      name: uploadName(query.get('name')),
+      size,
+    }
+  );
+};

package/js/src/server/routes-automations.js

@@ -199,6 +199,20 @@ const sendOnce = (ctx, draft, send) => {
   return promise;
 };
 
+// The network has it: a store that cannot take the copy (a full disk,
+// R-Z14) does not make it unsent — answering "failed" would invite a
+// second send. The sync brings the message once the store takes writes.
+const keepSent = async (store, links) => {
+  try {
+    for (const link of links) {
+      await store.put(link);
+    }
+    return true;
+  } catch {
+    return false;
+  }
+};
+
 const handleSend = async (store, res, body, ctx) => {
   if (!validChat(body) || !String(body.text ?? '').trim()) {
     return fail(res, 400, 'bad-message', 'source, chat and text are needed');
@@ -232,14 +246,13 @@ const handleSend = async (store, res, body, ctx) => {
     const sent = await sendOnce(ctx, draft, async () => {
       const result = await ctx.send(draft);
       const links = sentLinks(draft, result);
-      for (const link of links) {
-        await store.put(link);
-      }
+      const stored = await keepSent(store, links);
       return {
         status: 'sent',
         via: result?.via,
         link: links.at(-1) ?? null,
         links,
+        ...(stored ? {} : { stored: false }),
         ...(result?.scheduled ? { scheduled: true } : {}),
       };
     });

package/js/src/server/routes-pull.js

@@ -105,10 +105,8 @@ export const createPullServices = (store, log, options = {}, github = null) => {
       pullRunner.start();
       vkInbox.start();
     },
-    stop: () => {
-      pullRunner.stop();
-      vkInbox.stop();
-    },
+    // Resolves once the pulls in flight have stored what they fetched.
+    stop: () => Promise.all([vkInbox.stop(), pullRunner.stop()]).then(() => {}),
   };
 };
 

package/js/src/sources/browser-sender.js

@@ -39,6 +39,7 @@ import { createAdapterSender } from '../crm/reengage.js';
 import { WEB_UIS } from './web-uis.js';
 import { createBrowserActions } from './browser-actions.js';
 import { createMessageActions } from './message-actions.js';
+import { markSendOutcome } from './http.js';
 
 const isNumericId = (value) => /^-?\d+$/.test(String(value));
 
@@ -227,16 +228,29 @@ const typeLines = async (commander, text, delayMs) => {
   }
 };
 
+// A failure while pressing Send may come after the press took effect: the
+// message may be out, so it is never retried or sent another way (R-Z4).
+const sentInDoubt = (error) =>
+  Object.assign(markSendOutcome(error, { ambiguous: true }), {
+    code: 'send-unknown',
+    outcome: 'unknown',
+    fallback: false,
+  });
+
 /** Click the send button when it is there, otherwise press the key. */
 const pressSend = async (ui, commander) => {
   const button = ui.send?.selector
     ? await firstMatch(commander, ui.send.selector)
     : null;
-  if (button) {
-    await commander.clickButton({ selector: button, scrollIntoView: true });
-    return;
+  try {
+    if (button) {
+      await commander.clickButton({ selector: button, scrollIntoView: true });
+      return;
+    }
+    await commander.pressKey({ key: ui.send?.key ?? 'Enter' });
+  } catch (error) {
+    throw sentInDoubt(error);
   }
-  await commander.pressKey({ key: ui.send?.key ?? 'Enter' });
 };
 
 /**
@@ -534,6 +548,22 @@ const markUnknown = (error) => {
   return error;
 };
 
+// What a failed web-UI send raises after the API failed with `apiError`.
+const browserFailure = (error, apiError, draft, action) => {
+  // "Not allowed to automate this" is the answer, not a pair of failures.
+  if (error?.code === 'unsupported') {
+    return error;
+  }
+  // Maybe delivered by the web UI: told as such, never "failed" (R-Z4).
+  if (!apiError || error?.ambiguous === true) {
+    return markUnknown(error);
+  }
+  return new SendFallbackError(
+    [apiError, error],
+    `${draft?.source}: ${action === 'send' ? 'sending' : action} failed through the API (${apiError.message}) and the web UI (${error.message})`
+  );
+};
+
 /**
  * API first, browser second.
  *
@@ -584,17 +614,7 @@ export const createFallbackSender =
         apiError ? { apiError: apiError.message } : {}
       );
     } catch (error) {
-      // "Not allowed to automate this" is the answer, not a pair of failures.
-      if (error?.code === 'unsupported') {
-        throw error;
-      }
-      if (!apiError) {
-        throw error;
-      }
-      throw new SendFallbackError(
-        [apiError, error],
-        `${draft?.source}: ${action === 'send' ? 'sending' : action} failed through the API (${apiError.message}) and the web UI (${error.message})`
-      );
+      throw browserFailure(error, apiError, draft, action);
     }
   };
 

package/js/src/sources/pull-runner.js

@@ -86,9 +86,12 @@ export const createPullRunner = ({
   const lastStart = new Map();
   const running = new Set();
   const timers = new Map();
+  // Runs in flight, so that `stop()` can be waited for: a server that
+  // closed mid-run would otherwise store the pull after its last flush.
+  const flights = new Set();
   let stopped = true;
 
-  const execute = async (source) => {
+  const runOnce = async (source) => {
     const job = jobs[source];
     running.add(source);
     lastStart.set(source, now());
@@ -115,6 +118,13 @@ export const createPullRunner = ({
     }
   };
 
+  const execute = (source) => {
+    const flight = runOnce(source);
+    flights.add(flight);
+    flight.finally(() => flights.delete(flight)).catch(() => {});
+    return flight;
+  };
+
   const fetchNow = async (source) => {
     const job = jobs[source];
     if (!job) {
@@ -181,12 +191,14 @@ export const createPullRunner = ({
         }
       }
     },
+    /** Stop the schedules; resolves once the runs in flight are done. */
     stop() {
       stopped = true;
       for (const timer of timers.values()) {
         clearTimer(timer);
       }
       timers.clear();
+      return Promise.allSettled([...flights]).then(() => {});
     },
     sources: () => Object.keys(jobs),
     connected: (source) => jobs[source].ready(),

package/js/src/sources/vk-longpoll.js

@@ -206,6 +206,9 @@ export const createVkLongPoll = ({
 }) => {
   let session = null;
   let looping = false;
+  // Each start's loop; a stopped one ends even if another has started.
+  let generation = 0;
+  let loopDone = Promise.resolve();
   let abort = new AbortController();
 
   const request = (state, seconds) =>
@@ -280,10 +283,11 @@ export const createVkLongPoll = ({
     return { ok: true, updates: updates.length, imported: links.length };
   };
 
-  const loop = async () => {
+  const loop = async (gen) => {
     let fails = 0;
+    const live = () => looping && gen === generation;
     await sleep(startDelayMs);
-    while (looping) {
+    while (live()) {
       try {
         if (await ready()) {
           const started = now();
@@ -297,7 +301,7 @@ export const createVkLongPoll = ({
           await sleep(idleMs);
         }
       } catch (error) {
-        if (!looping) {
+        if (!live()) {
           return;
         }
         fails += 1;
@@ -332,13 +336,16 @@ export const createVkLongPoll = ({
     start() {
       if (!looping) {
         looping = true;
-        loop();
+        generation += 1;
+        loopDone = loop(generation).catch(() => {});
       }
     },
+    /** Resolves once a batch in flight is stored (no write after close). */
     stop() {
       looping = false;
       abort.abort();
       cancelPauses();
+      return loopDone;
     },
   };
 };

package/js/src/sources/whatsapp-file.js

@@ -51,58 +51,57 @@ const withMedia = async (links, ctx, counts) => {
   return out;
 };
 
+// The chat, and its media copied out, of an export zip already on disk.
+const readZipFile = async (name, file, mediaDir) => {
+  const zip = await openZipFile(file);
+  try {
+    const chat = textEntry(zip.entries);
+    if (!chat) {
+      throw new Error('no chat .txt found in this zip');
+    }
+    const archive = {
+      text: (await chat.read()).toString('utf8'),
+      fileName: /whatsapp/i.test(chat.name) ? chat.name : name,
+      files: zip.entries.map((e) => e.name),
+    };
+    const ctx = createArchiveContext({
+      network: 'whatsapp',
+      entries: zip.entries,
+      save: mediaDir ? mediaSaver(mediaDir) : null,
+    });
+    const counts = { saved: 0, missing: 0 };
+    const links = await withMedia(
+      await getSource('whatsapp').parseArchive(archive),
+      ctx,
+      counts
+    );
+    return { links, media: { ...counts, warnings: ctx.warnings } };
+  } finally {
+    await zip.close();
+  }
+};
+
 const readZip = async (name, bytes, mediaDir) => {
   const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ms-whatsapp-'));
   const file = path.join(dir, 'export.zip');
   try {
     await fs.writeFile(file, bytes, { mode: 0o600 });
-    const zip = await openZipFile(file);
-    try {
-      const chat = textEntry(zip.entries);
-      if (!chat) {
-        throw new Error('no chat .txt found in this zip');
-      }
-      const archive = {
-        text: (await chat.read()).toString('utf8'),
-        fileName: /whatsapp/i.test(chat.name) ? chat.name : name,
-        files: zip.entries.map((e) => e.name),
-      };
-      const ctx = createArchiveContext({
-        network: 'whatsapp',
-        entries: zip.entries,
-        save: mediaDir ? mediaSaver(mediaDir) : null,
-      });
-      const counts = { saved: 0, missing: 0 };
-      const links = await withMedia(
-        await getSource('whatsapp').parseArchive(archive),
-        ctx,
-        counts
-      );
-      return { links, media: { ...counts, warnings: ctx.warnings } };
-    } finally {
-      await zip.close();
-    }
+    return await readZipFile(name, file, mediaDir);
   } finally {
     await fs.rm(dir, { recursive: true, force: true });
   }
 };
 
-export const importWhatsAppFile = async (
-  store,
-  name,
-  bytes,
-  { mediaDir = null } = {}
-) => {
-  const { links, media } = isZip(name, bytes)
-    ? await readZip(name, bytes, mediaDir)
-    : {
-        links: await getSource('whatsapp').parseArchive({
-          text: bytes.toString('utf8'),
-          fileName: name,
-          files: [],
-        }),
-        media: null,
-      };
+const readText = async (name, bytes) => ({
+  links: await getSource('whatsapp').parseArchive({
+    text: bytes.toString('utf8'),
+    fileName: name,
+    files: [],
+  }),
+  media: null,
+});
+
+const storeLinks = async (store, { links, media }) => {
   for (const link of links) {
     await store.put(stampSourceLink(link, 'whatsapp'));
   }
@@ -112,3 +111,41 @@ export const importWhatsAppFile = async (
     ...(media ? { media } : {}),
   };
 };
+
+export const importWhatsAppFile = async (
+  store,
+  name,
+  bytes,
+  { mediaDir = null } = {}
+) =>
+  storeLinks(
+    store,
+    isZip(name, bytes)
+      ? await readZip(name, bytes, mediaDir)
+      : await readText(name, bytes)
+  );
+
+/**
+ * The same for a file already on disk (an upload streamed to a temp file,
+ * R-Z23): a zip is read in place, never loaded whole.
+ */
+export const importWhatsAppPath = async (
+  store,
+  name,
+  file,
+  { mediaDir = null } = {}
+) => {
+  const handle = await fs.open(file, 'r');
+  const head = Buffer.alloc(4);
+  try {
+    await handle.read(head, 0, 4, 0);
+  } finally {
+    await handle.close();
+  }
+  return storeLinks(
+    store,
+    isZip(name, head)
+      ? await readZipFile(name, file, mediaDir)
+      : await readText(name, await fs.readFile(file))
+  );
+};