meta-sovereign 0.49.8 → 0.49.10

package/js/src/server/index.js

@@ -39,6 +39,7 @@
  *   POST /api/email/pull           -> local-server email API/protocol proxy
  *   POST /api/email/send           -> local-server email send proxy
  *   GET  /api/email/attachment     -> a mail attachment from the media folder (routes-email.js)
+ *   GET  /api/media/<network>/<sha256>[.ext] -> imported archive media (routes-media.js, R-Z15)
  *   GET  /api/backups              -> list archives (gated on archiveDir)
  *   POST /api/backups              -> { passphrase?, keep? } -> { file }
  *   POST /api/backups/restore      -> { file, passphrase? } -> { restored }
@@ -95,6 +96,7 @@ import { profileDirIn } from '../sources/whatsapp-web-signin.js';
 import { handleTelegramRoutes } from './routes-telegram.js';
 import { handleMessageRoutes } from './routes-messages.js';
 import { handleEmailGets } from './routes-email.js';
+import { handleMediaGets } from './routes-media.js';
 import { createAttachmentStore } from './email-attachments.js';
 import { createEmailPoller } from './email-poller.js';
 import { setAttachmentSink } from '../sources/email.js';
@@ -586,6 +588,7 @@ const routeOn = async (store, req, res, ctx, url, p) => {
     () => handleTelegramRoutes(req, res, p, ctx),
     () => handleMessageRoutes(store, req, res, p, url, ctx),
     () => handleEmailGets(store, req, res, p, url, ctx),
+    () => handleMediaGets(req, res, p, ctx),
     () => handleDeletedRoutes(store, req, res, p, url, ctx),
     () => handleAudienceSyncRoute(store, req, res, p, ctx),
     () => handleChatRoutes(req, res, p, url, ctx),

package/js/src/server/routes-media.js

@@ -0,0 +1,179 @@
+/**
+ * Imported archive media (R-Z15):
+ *   GET /api/media/<network>/<sha256>[.<ext>]
+ * serves a file that an archive import (R-Z10 Facebook/Messenger,
+ * Instagram, X, LinkedIn; R-Z15 WhatsApp) copied to
+ * `<store>/media/<network>/<sha256><ext>`. The path is built only from
+ * validated parts (a fixed network list, 64 hex digits, a fixed extension
+ * list), so nothing outside that folder can be named. Pictures, audio and
+ * video are shown inline; everything else, svg and html included, is
+ * offered as a download with a sandboxing CSP. Audio and video answer
+ * `Range`. The file is streamed, never read whole.
+ */
+
+import { createReadStream, promises as fs } from 'node:fs';
+import path from 'node:path';
+import { fail } from './util.js';
+import { fromApp } from './routes-telegram-files.js';
+
+export const MEDIA_NETWORKS = Object.freeze([
+  'facebook',
+  'messenger',
+  'instagram',
+  'x',
+  'linkedin',
+  'whatsapp',
+]);
+
+const INLINE = {
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  png: 'image/png',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  mp4: 'video/mp4',
+  m4v: 'video/mp4',
+  mov: 'video/quicktime',
+  webm: 'video/webm',
+  mp3: 'audio/mpeg',
+  m4a: 'audio/mp4',
+  aac: 'audio/aac',
+  ogg: 'audio/ogg',
+  opus: 'audio/ogg',
+  oga: 'audio/ogg',
+  wav: 'audio/wav',
+};
+// Downloaded, never shown: documents, archives, and the types a browser
+// would run (svg, html), which are sent as opaque bytes.
+const OPAQUE = 'application/octet-stream';
+const DOWNLOAD = {
+  pdf: 'application/pdf',
+  txt: 'text/plain; charset=utf-8',
+  vcf: 'text/vcard; charset=utf-8',
+  csv: 'text/csv; charset=utf-8',
+  json: 'application/json',
+  zip: 'application/zip',
+  ...Object.fromEntries(
+    [
+      'doc',
+      'docx',
+      'xls',
+      'xlsx',
+      'ppt',
+      'pptx',
+      'heic',
+      'heif',
+      '3gp',
+      'amr',
+      'mkv',
+      'svg',
+      'html',
+      'htm',
+      'xml',
+      'js',
+    ].map((ext) => [ext, OPAQUE])
+  ),
+};
+
+export const MEDIA_EXTENSIONS = Object.freeze([
+  ...Object.keys(INLINE),
+  ...Object.keys(DOWNLOAD),
+]);
+
+const URL_SHAPE =
+  /^\/api\/media\/([a-z]{1,16})\/([0-9a-f]{64})(?:\.([a-z0-9]{1,8}))?$/;
+const IN_MEDIA = /^\/api\/media(\/|$)/;
+
+const refuse = (res, status, code, error) =>
+  fail(res, status, code, error) ?? true;
+const notFound = (res) => refuse(res, 404, 'not-found', 'no such media file');
+
+/** `{network, sha256, ext}` of a valid media path, else null. */
+export const parseMediaPath = (p) => {
+  const m = URL_SHAPE.exec(p);
+  if (!m || !MEDIA_NETWORKS.includes(m[1])) {
+    return null;
+  }
+  const ext = m[3] ?? '';
+  return ext && !MEDIA_EXTENSIONS.includes(ext)
+    ? null
+    : { network: m[1], sha256: m[2], ext };
+};
+
+const rangeOf = (header, size) => {
+  const m = /^bytes=(\d*)-(\d*)$/.exec(String(header ?? ''));
+  if (!m || (m[1] === '' && m[2] === '')) {
+    return null;
+  }
+  const start = Math.max(0, m[1] === '' ? size - Number(m[2]) : Number(m[1]));
+  const end =
+    m[1] === '' || m[2] === '' ? size - 1 : Math.min(Number(m[2]), size - 1);
+  return start <= end && start < size
+    ? { start, end }
+    : { unsatisfiable: true };
+};
+
+const send = ({ req, res, file, ext, size }) => {
+  const inline = ext in INLINE;
+  const headers = {
+    'content-type': INLINE[ext] ?? DOWNLOAD[ext] ?? OPAQUE,
+    'x-content-type-options': 'nosniff',
+    'content-disposition': inline
+      ? 'inline'
+      : `attachment; filename="media${ext ? `.${ext}` : ''}"`,
+    'cache-control': 'private, max-age=3600',
+    'content-security-policy': "sandbox; default-src 'none'",
+    'accept-ranges': inline ? 'bytes' : 'none',
+  };
+  const range = inline ? rangeOf(req.headers.range, size) : null;
+  if (range?.unsatisfiable) {
+    res.writeHead(416, { ...headers, 'content-range': `bytes */${size}` });
+    res.end();
+    return true;
+  }
+  const part = range ?? { start: 0, end: size - 1 };
+  res.writeHead(range ? 206 : 200, {
+    ...headers,
+    'content-length': size === 0 ? 0 : part.end - part.start + 1,
+    ...(range
+      ? { 'content-range': `bytes ${part.start}-${part.end}/${size}` }
+      : {}),
+  });
+  if (req.method === 'HEAD' || size === 0) {
+    res.end();
+    return true;
+  }
+  const stream = createReadStream(file, part);
+  stream.on('error', () => res.destroy());
+  stream.pipe(res);
+  return true;
+};
+
+/** GET /api/media/... (see the header). */
+export const handleMediaGets = async (req, res, p, ctx) => {
+  if (!IN_MEDIA.test(p) || (req.method !== 'GET' && req.method !== 'HEAD')) {
+    return false;
+  }
+  if (!fromApp(req, ctx?.allowOrigins)) {
+    return refuse(res, 403, 'forbidden', 'media loads only in the app');
+  }
+  const wanted = parseMediaPath(p);
+  if (!wanted || !ctx?.storeDir) {
+    return notFound(res);
+  }
+  const { network, sha256, ext } = wanted;
+  const file = path.join(
+    ctx.storeDir,
+    'media',
+    network,
+    `${sha256}${ext ? `.${ext}` : ''}`
+  );
+  try {
+    const info = await fs.lstat(file);
+    return info.isFile()
+      ? send({ req, res, file, ext, size: info.size })
+      : notFound(res);
+  } catch {
+    return notFound(res);
+  }
+};

package/js/src/server/routes-whatsapp.js

@@ -6,16 +6,19 @@
  *   POST /api/whatsapp/web/close       -> closes it
  *   POST /api/whatsapp/archive {name, data(base64)}
  *                                      -> imports an "Export chat" .txt or .zip
+ *                                         (a zip's photos/videos/files are kept in media/whatsapp, R-Z15)
  *
  * The sign-in window is for the person to scan the QR code themselves
  * (see ../sources/whatsapp-web-signin.js); these routes never touch it.
  */
 
+import path from 'node:path';
 import { json, readBody } from './util.js';
 import { isTrustedMutation } from './routes-auto-import.js';
 import { safeMessage } from './redact.js';
 import { createWhatsAppWebSignIn } from '../sources/whatsapp-web-signin.js';
 import { importWhatsAppFile } from '../sources/whatsapp-file.js';
+import { isFatalWrite } from '../sources/archive/context.js';
 
 const PREFIX = '/api/whatsapp/';
 
@@ -43,7 +46,7 @@ const failure = (res, error) =>
     code: error.code ?? 'internal',
   });
 
-const archiveRoute = async (store, req, res) => {
+const archiveRoute = async (store, req, res, ctx) => {
   const body = await readBody(req, { limit: 64 * 1024 * 1024 });
   const bytes = Buffer.from(String(body.data ?? ''), 'base64');
   if (bytes.length === 0 || typeof body.name !== 'string') {
@@ -53,9 +56,21 @@ const archiveRoute = async (store, req, res) => {
     });
   }
   try {
-    return reply(res, 200, await importWhatsAppFile(store, body.name, bytes));
+    // Media files of a zip are kept next to the store (R-Z15).
+    const mediaDir = ctx?.storeDir ? path.join(ctx.storeDir, 'media') : null;
+    return reply(
+      res,
+      200,
+      await importWhatsAppFile(store, body.name, bytes, { mediaDir })
+    );
   } catch (error) {
-    return reply(res, 400, { error: safeMessage(error), code: 'bad-archive' });
+    // A full or unwritable disk is not a bad file (R-Z14).
+    return isFatalWrite(error)
+      ? reply(res, 507, {
+          error: safeMessage(error),
+          code: 'store-write-failed',
+        })
+      : reply(res, 400, { error: safeMessage(error), code: 'bad-archive' });
   }
 };
 
@@ -94,7 +109,7 @@ export const handleWhatsAppRoutes = async (store, req, res, p, ctx) => {
     });
   }
   if (isArchive) {
-    return req.method === 'POST' ? archiveRoute(store, req, res) : false;
+    return req.method === 'POST' ? archiveRoute(store, req, res, ctx) : false;
   }
   const expected = route === 'web' ? 'GET' : 'POST';
   return req.method === expected

package/js/src/sources/archive/media-store.js

@@ -8,6 +8,7 @@
 import { createHash, randomBytes } from 'node:crypto';
 import { createWriteStream, promises as fs } from 'node:fs';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import { pipeline } from 'node:stream/promises';
 
 const extensionOf = (name) => {
@@ -39,7 +40,7 @@ export const saveMedia = async ({ mediaDir, network, stream, name }) => {
     );
     const sha256 = hash.digest('hex');
     const file = `${sha256}${extensionOf(name)}`;
-    await fs.rename(temp, path.join(dir, file));
+    await renameDurable(temp, path.join(dir, file)); // (R-Z19)
     await fs.chmod(path.join(dir, file), 0o600);
     return { path: `media/${network}/${file}`, sha256, size };
   } catch (error) {

package/js/src/sources/auto-import/sign-ins.js

@@ -28,6 +28,7 @@
 import { promises as fs } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import { execFile } from 'node:child_process';
 import { autoImportRecipes } from './recipes.js';
 
@@ -229,7 +230,7 @@ export const createSignIns = ({
     const file = fileOf(provider);
     const temp = `${file}.${process.pid}.tmp`;
     await fs.writeFile(temp, JSON.stringify(seed), { mode: 0o600 });
-    await fs.rename(temp, file);
+    await renameDurable(temp, file); // (R-Z19)
   };
 
   const access = () => probeFullDiskAccess({ platform, homeDir, env });

package/js/src/sources/auto-import/telegram-creds.js

@@ -32,6 +32,7 @@
 import { promises as fs } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import crypto from 'node:crypto';
 import { timedRequest } from '../http.js';
 
@@ -765,7 +766,7 @@ export const createTelegramCreds = ({
       typeof v === 'bigint' ? v.toString() : v
     );
     await fs.writeFile(temp, text, { mode: 0o600 });
-    await fs.rename(temp, cacheFile);
+    await renameDurable(temp, cacheFile); // (R-Z19)
   };
 
   const verifyAll = (records) =>

package/js/src/sources/telegram-export-run.js

@@ -10,6 +10,7 @@
 
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
+import { renameDurable } from '../storage/durable.js';
 import { rawMessage, floodSeconds } from './telegram-history.js';
 import { bigSize, locationOf, fileLocation } from './telegram-media.js';
 import { failureOf } from './telegram-stickers.js';
@@ -149,7 +150,7 @@ const download = async (r, info, target) => {
         outputFile: part,
       })
     );
-    await fs.rename(part, target);
+    await renameDurable(part, target); // (R-Z19)
   } catch (error) {
     await fs.rm(part, { force: true });
     throw error;

package/js/src/sources/whatsapp-archive.js

@@ -154,6 +154,11 @@ const KINDS = [
 const kindOf = (name) =>
   KINDS.find(([, re]) => re.test(name))?.[0] ?? 'document';
 
+// The page's network-neutral attachment kinds (R-Z15); `type` stays as
+// stored by R-Z9 so earlier imports and later ones read alike.
+const PAGE_KIND = { image: 'photo', document: 'file', contact: 'file' };
+const pageKind = (type) => PAGE_KIND[type] ?? type;
+
 const lookupFile = (files, name) => {
   const base = (f) => String(f).split(/[\\/]/).pop();
   return (
@@ -169,7 +174,13 @@ const mediaOf = (body, files) => {
   const line = first.trim();
   if (OMITTED.test(line)) {
     return {
-      attachment: { type: 'media', name: null, file: null, omitted: true },
+      attachment: {
+        type: 'media',
+        kind: 'file',
+        name: null,
+        file: null,
+        omitted: true,
+      },
       caption: more.join('\n').trim(),
     };
   }
@@ -179,8 +190,9 @@ const mediaOf = (body, files) => {
     return null;
   }
   const file = lookupFile(files, name);
+  const type = kindOf(name);
   return {
-    attachment: { type: kindOf(name), name, file, omitted: false },
+    attachment: { type, kind: pageKind(type), name, file, omitted: false },
     caption: more.join('\n').trim(),
   };
 };

package/js/src/sources/whatsapp-file.js

@@ -1,12 +1,20 @@
 /**
- * Import one WhatsApp "Export chat" file that arrived as bytes (R-Z9):
- * the `.txt`, or the `.zip` of an export with media. The text is parsed
- * by ./whatsapp-archive.js; the zip's other entries are only used to map
- * `<attached: ...>` names to files inside the zip (the media bytes are
- * not copied into the store).
+ * Import one WhatsApp "Export chat" file that arrived as bytes (R-Z9,
+ * R-Z15): the `.txt`, or the `.zip` of an export with media. The text is
+ * parsed by ./whatsapp-archive.js. A zip is written to a private temp file
+ * and read with the streaming reader (./archive/zip-file.js); each file
+ * the chat names (`<attached: f>`, `f (file attached)`) is copied into
+ * `<store>/media/whatsapp/<sha256>.<ext>` (0600) and the attachment gets
+ * `path`, `sha256` and `size`, so the page can show it. Without a
+ * `mediaDir` the names are mapped but no bytes are kept.
  */
 
-import { readZipEntries } from './auto-import/archive.js';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createArchiveContext } from './archive/context.js';
+import { mediaSaver } from './archive/media-store.js';
+import { openZipFile } from './archive/zip-file.js';
 import { getSource, stampSourceLink } from './index.js';
 
 const ZIP_LOCAL_HEADER = 0x04034b50;
@@ -19,23 +27,88 @@ const textEntry = (entries) =>
   entries.find((e) => /(^|\/)_chat\.txt$/i.test(e.name)) ??
   entries.find((e) => /\.txt$/i.test(e.name));
 
-export const importWhatsAppFile = async (store, name, bytes) => {
-  let archive = { text: bytes.toString('utf8'), fileName: name, files: [] };
-  if (isZip(name, bytes)) {
-    const entries = readZipEntries(bytes);
-    const chat = textEntry(entries);
-    if (!chat) {
-      throw new Error('no chat .txt found in this zip');
+// Copy each attachment's file into media/ and say where it went.
+const withMedia = async (links, ctx, counts) => {
+  const out = [];
+  for (const link of links) {
+    const attachments = [];
+    for (const attachment of link.attachments ?? []) {
+      const copy = attachment.file ? await ctx.media(attachment.file) : null;
+      counts[copy ? 'saved' : 'missing'] += attachment.omitted ? 0 : 1;
+      attachments.push(
+        copy
+          ? {
+              ...attachment,
+              path: copy.path,
+              sha256: copy.sha256,
+              size: copy.size,
+            }
+          : attachment
+      );
     }
-    archive = {
-      text: chat.read().toString('utf8'),
-      fileName: /whatsapp/i.test(chat.name) ? chat.name : name,
-      files: entries.map((e) => e.name),
-    };
+    out.push(attachments.length ? { ...link, attachments } : link);
   }
-  const links = await getSource('whatsapp').parseArchive(archive);
+  return out;
+};
+
+const readZip = async (name, bytes, mediaDir) => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ms-whatsapp-'));
+  const file = path.join(dir, 'export.zip');
+  try {
+    await fs.writeFile(file, bytes, { mode: 0o600 });
+    const zip = await openZipFile(file);
+    try {
+      const chat = textEntry(zip.entries);
+      if (!chat) {
+        throw new Error('no chat .txt found in this zip');
+      }
+      const archive = {
+        text: (await chat.read()).toString('utf8'),
+        fileName: /whatsapp/i.test(chat.name) ? chat.name : name,
+        files: zip.entries.map((e) => e.name),
+      };
+      const ctx = createArchiveContext({
+        network: 'whatsapp',
+        entries: zip.entries,
+        save: mediaDir ? mediaSaver(mediaDir) : null,
+      });
+      const counts = { saved: 0, missing: 0 };
+      const links = await withMedia(
+        await getSource('whatsapp').parseArchive(archive),
+        ctx,
+        counts
+      );
+      return { links, media: { ...counts, warnings: ctx.warnings } };
+    } finally {
+      await zip.close();
+    }
+  } finally {
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+};
+
+export const importWhatsAppFile = async (
+  store,
+  name,
+  bytes,
+  { mediaDir = null } = {}
+) => {
+  const { links, media } = isZip(name, bytes)
+    ? await readZip(name, bytes, mediaDir)
+    : {
+        links: await getSource('whatsapp').parseArchive({
+          text: bytes.toString('utf8'),
+          fileName: name,
+          files: [],
+        }),
+        media: null,
+      };
   for (const link of links) {
     await store.put(stampSourceLink(link, 'whatsapp'));
   }
-  return { imported: links.length, chat: links[0]?.chat ?? null };
+  return {
+    imported: links.length,
+    chat: links[0]?.chat ?? null,
+    ...(media ? { media } : {}),
+  };
 };

package/js/src/storage/durable.js

@@ -45,6 +45,46 @@ export const fsyncDir = async (dir) => {
   }
 };
 
+const WINDOWS_BUSY = new Set(['EPERM', 'EBUSY', 'EACCES']);
+const defaultSleep = (ms) =>
+  new Promise((resolve) => {
+    globalThis.setTimeout(resolve, ms);
+  });
+
+/**
+ * Rename `from` over `to`, retrying while another handle holds the target
+ * (R-Z19). On Windows a rename fails with EPERM/EBUSY/EACCES while any
+ * process (antivirus, the search indexer, our own snapshot worker) has the
+ * target open, usually for a few milliseconds: retried up to `attempts`
+ * times with a growing pause (about a second in all, like graceful-fs).
+ * Elsewhere rename is atomic and those codes are real errors, so only
+ * EBUSY is retried. After the last attempt the error is thrown and the
+ * target is untouched. `rename`, `platform` and `sleep` are injectable.
+ */
+export const renameDurable = async (
+  from,
+  to,
+  {
+    rename = fs.rename,
+    platform = process.platform,
+    sleep = defaultSleep,
+    attempts = 10,
+    delayMs = 20,
+  } = {}
+) => {
+  const codes = platform === 'win32' ? WINDOWS_BUSY : new Set(['EBUSY']);
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await rename(from, to);
+    } catch (error) {
+      if (attempt >= attempts || !codes.has(error?.code)) {
+        throw error;
+      }
+      await sleep(delayMs * attempt);
+    }
+  }
+};
+
 /** Create `dir` (and parents) private to the user. */
 export const ensurePrivateDir = (dir) =>
   fs.mkdir(dir, { recursive: true, mode: DIR_MODE });
@@ -74,7 +114,7 @@ export const tighten = async (target, mode = FILE_MODE) => {
 export const writeFileDurable = async (
   file,
   data,
-  { utime, tag = '' } = {}
+  { utime, tag = '', rename } = {}
 ) => {
   const dir = path.dirname(file);
   await ensurePrivateDir(dir);
@@ -90,7 +130,7 @@ export const writeFileDurable = async (
     } finally {
       await handle.close();
     }
-    await fs.rename(tmp, file);
+    await renameDurable(tmp, file, { rename });
   } catch (error) {
     // A full disk: the half-written temp file goes at once, not at the
     // next start (R-Z14).
@@ -116,7 +156,7 @@ export const writeStreamDurable = async (file, ...streams) => {
       createWriteStream(tmp, { flags: 'w', mode: FILE_MODE })
     );
     await fsyncFile(tmp);
-    await fs.rename(tmp, file);
+    await renameDurable(tmp, file);
   } catch (error) {
     await fs.unlink(tmp).catch(() => {});
     throw error;

package/js/src/storage/lino-store.js

@@ -28,7 +28,13 @@ import { createInterface } from 'node:readline';
 import { decodeLine, encodeLine } from 'lino-objects-codec';
 import { createMemoryStore } from './universal.js';
 import { parseLino } from './lino.js';
-import { FILE_MODE, fsyncDir, fsyncFile, tighten } from './durable.js';
+import {
+  FILE_MODE,
+  fsyncDir,
+  fsyncFile,
+  renameDurable,
+  tighten,
+} from './durable.js';
 import { enoughSpace, skippedEvent } from './disk-space.js';
 import { createWriteQueue, storeError, writeFailed } from './write-queue.js';
 
@@ -380,7 +386,7 @@ export const readLinoFile = async (filePath) => {
 // Write every link to a temp file in chunks, flush it to disk, swap it
 // in, and flush the directory: after a power cut the file is the old log
 // or the whole new one, never an empty or half-written one (R-Z2).
-const rewriteFile = async (filePath, links) => {
+const rewriteFile = async (filePath, links, rename) => {
   await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
   const tmp = `${filePath}.${process.pid}.tmp`;
   try {
@@ -393,7 +399,7 @@ const rewriteFile = async (filePath, links) => {
     } finally {
       await handle.close();
     }
-    await fs.rename(tmp, filePath);
+    await renameDurable(tmp, filePath, { rename });
   } catch (error) {
     // The half-written copy goes at once: on a full disk it is what
     // keeps the disk full (R-Z14).
@@ -461,6 +467,9 @@ const SYNC_MS = 200;
 
 // A compaction skipped for lack of space is asked again after this long (R-Z14).
 const COMPACT_RETRY_MS = 10 * 60_000;
+// A compaction that failed (a busy file on Windows) leaves the old log intact
+// and is asked again after this, not on every batch (R-Z19).
+const COMPACT_FAILED_RETRY_MS = 30_000;
 
 // Flushes appends to disk together, `syncMs` after the first (R-Z2).
 const createSyncer = (filePath, syncMs, enqueue) => {
@@ -506,7 +515,15 @@ const compactionFailed = (onEvent) => (error) =>
 // The log rewritten with one line per live link, after the unreadable
 // lines were set aside (R-Z2); skipped when the disk has no room for the
 // copy (R-Z14). `{ skipped }`.
-const compactLog = async ({ filePath, bad, mem, onCorrupt, onEvent, free }) => {
+const compactLog = async ({
+  filePath,
+  bad,
+  mem,
+  onCorrupt,
+  onEvent,
+  free,
+  rename,
+}) => {
   if (bad.overflow) {
     return { skipped: false }; // more unreadable lines than can be kept
   }
@@ -522,7 +539,7 @@ const compactLog = async ({ filePath, bad, mem, onCorrupt, onEvent, free }) => {
     bad.lines = [];
     bad.count = 0;
   }
-  await rewriteFile(filePath, await mem.query());
+  await rewriteFile(filePath, await mem.query(), rename);
   return { skipped: false };
 };
 
@@ -588,6 +605,14 @@ const createHold = () => {
   };
 };
 
+// A compaction skipped for lack of space, as an error for the write queue.
+const throwingWhenSkipped = (compact) => async () => {
+  const done = await compact();
+  if (done?.skipped) {
+    throw storeError('ENOSPC', 'not enough free space to rewrite the log');
+  }
+};
+
 const readOnlyError = () =>
   Object.assign(new Error('the store was opened read-only'), {
     code: 'STORE_READ_ONLY',
@@ -607,6 +632,7 @@ export const createLinoTextStore = async (
     // (tests inject an append handle that fails).
     queue: queueOptions = {},
     measureFree,
+    rename, // injectable fs.rename (R-Z19)
   } = {}
 ) => {
   const mem = createMemoryStore();
@@ -642,6 +668,7 @@ export const createLinoTextStore = async (
       onCorrupt,
       onEvent,
       free: measureFree,
+      rename,
     });
     if (done.skipped) {
       compactAfterAt = Date.now() + COMPACT_RETRY_MS;
@@ -676,16 +703,9 @@ export const createLinoTextStore = async (
     attempt: () => (rewrite ? compactOrThrow() : wq.write()),
     onRecovered: syncer.soon,
   });
-  const compactOrThrow = async () => {
-    const done = await compact();
-    if (done?.skipped) {
-      throw storeError('ENOSPC', 'not enough free space to rewrite the log');
-    }
-  };
+  const compactOrThrow = throwingWhenSkipped(compact);
   const hold = createHold();
   const maintain = async () => {
-    // The batch is on disk by now; a failed compaction costs nothing but
-    // the rewrite, so it never fails the writes (R-Z14).
     try {
       if (
         !hold.active() &&
@@ -695,6 +715,7 @@ export const createLinoTextStore = async (
         await compact();
       }
     } catch (error) {
+      compactAfterAt = Date.now() + COMPACT_FAILED_RETRY_MS;
       compactionFailed(onEvent)(error);
     }
   };

package/js/src/web/app.css

@@ -12556,3 +12556,14 @@ a.remote-chip:hover {
 main[data-view='chat'] > .api-error-toast {
   margin: 0;
 }
+
+/* R-Z15: attachments of imported archive messages (pictures, players,
+   file chips, "media not imported"). */
+.archive-media audio,
+.archive-media video {
+  max-width: 100%;
+}
+.remote-chip.is-missing {
+  color: var(--text-muted);
+  border-style: dashed;
+}

package/js/src/web/archive-media-model.js

@@ -0,0 +1,28 @@
+// What the page needs to know about an imported archive attachment
+// (R-Z15); pure, so it is tested without a browser.
+
+// `media/<network>/<64 hex>[.ext]`, the only paths the importers write.
+const STORED = /^media\/([a-z]{1,16})\/([0-9a-f]{64}(?:\.[a-z0-9]{1,8})?)$/;
+
+// What older WhatsApp imports stored: `type`, no `kind`.
+const LEGACY_KIND = { image: 'photo', document: 'file', contact: 'file' };
+
+/** The server path (`/api/media/...`) of a stored attachment, or null. */
+export const mediaPathOf = (attachment) => {
+  const m = STORED.exec(String(attachment?.path ?? ''));
+  return m ? `/api/media/${m[1]}/${m[2]}` : null;
+};
+
+/** One attachment as the page draws it. */
+export const archiveItem = (attachment) => {
+  const a = attachment ?? {};
+  return {
+    kind: a.kind ?? LEGACY_KIND[a.type] ?? a.type ?? 'file',
+    name: a.name ?? null,
+    size: Number(a.size) || 0,
+    path: mediaPathOf(a),
+    url: /^https?:\/\//.test(a.url ?? '') ? a.url : null,
+    omitted: Boolean(a.omitted),
+    caption: a.caption ?? null,
+  };
+};

package/js/src/web/chat-archive-media.js

@@ -0,0 +1,143 @@
+// Attachments of imported archive messages (R-Z15): Facebook/Messenger,
+// Instagram, X, LinkedIn and WhatsApp exports. The importers copy each
+// file to `media/<network>/<sha256>.<ext>` and store `{kind, name, path,
+// sha256, size}` on the message; the server shows it at
+// /api/media/<network>/<file> (../server/routes-media.js). Pictures are
+// drawn inline, audio and video get a player, other files are chips with
+// name and size; a file that was not imported (the archive did not hold
+// it, or WhatsApp omitted it) is a "media not imported" chip, never a
+// broken picture.
+
+import React, { useEffect, useState } from 'react';
+import { useT } from './i18n.js';
+import { mediaSrc } from './media-src.js';
+import { formatSize } from './download/model.js';
+
+const el = React.createElement;
+
+// The address the page loads the file from (the app's server).
+const useMediaUrl = (api, path) => {
+  const [url, setUrl] = useState(null);
+  useEffect(() => {
+    let live = true;
+    if (!path) {
+      return undefined;
+    }
+    const wanted =
+      typeof api?.archiveMediaUrl === 'function'
+        ? api.archiveMediaUrl(path)
+        : Promise.resolve(path);
+    wanted.then((got) => live && setUrl(got ?? null)).catch(() => {});
+    return () => {
+      live = false;
+    };
+  }, [api, path]);
+  return url;
+};
+
+const Missing = ({ item }) => {
+  const t = useT();
+  return el(
+    'span',
+    {
+      className: 'remote-chip is-missing',
+      'data-archive-missing': item.kind,
+      title: t('chat.archive.notImportedShort'),
+    },
+    t('chat.archive.notImported', { name: item.name ?? t('chat.remote.file') })
+  );
+};
+
+const FileChip = ({ item, url }) => {
+  const t = useT();
+  const label = [item.name ?? t('chat.remote.file'), formatSize(item.size)]
+    .filter(Boolean)
+    .join(' · ');
+  return el(
+    'a',
+    {
+      className: 'remote-chip',
+      'data-archive-file': item.kind,
+      href: url,
+      download: item.name ?? '',
+      title: t('chat.archive.save', { name: item.name ?? '' }),
+    },
+    `📎 ${label}`
+  );
+};
+
+const PICTURES = new Set(['photo', 'gif', 'sticker']);
+
+const Stored = ({ item, url }) => {
+  const [broken, setBroken] = useState(false);
+  if (!url) {
+    return null;
+  }
+  const label = item.name ?? item.kind;
+  if (PICTURES.has(item.kind) && !broken) {
+    return el(
+      'a',
+      { href: url, target: '_blank', rel: 'noopener noreferrer', title: label },
+      el('img', {
+        ...mediaSrc(url),
+        className: `remote-img remote-${item.kind}`,
+        alt: label,
+        loading: 'lazy',
+        decoding: 'async',
+        'data-archive-image': '',
+        onError: () => setBroken(true),
+      })
+    );
+  }
+  if (item.kind === 'video' && !broken) {
+    return el('video', {
+      ...mediaSrc(url),
+      className: 'remote-img',
+      controls: true,
+      preload: 'metadata',
+      'data-archive-video': '',
+      onError: () => setBroken(true),
+    });
+  }
+  if ((item.kind === 'audio' || item.kind === 'voice') && !broken) {
+    return el('audio', {
+      ...mediaSrc(url),
+      controls: true,
+      preload: 'none',
+      'data-archive-audio': '',
+      onError: () => setBroken(true),
+    });
+  }
+  return el(FileChip, { item, url });
+};
+
+const Item = ({ item, api }) => {
+  const url = useMediaUrl(api, item.path);
+  if (item.kind === 'link' && item.url) {
+    return el(
+      'a',
+      {
+        className: 'remote-chip',
+        href: item.url,
+        target: '_blank',
+        rel: 'noopener noreferrer',
+      },
+      item.caption || item.url
+    );
+  }
+  if (item.omitted || !item.path) {
+    return el(Missing, { item });
+  }
+  return el(Stored, { item, url });
+};
+
+/** All the attachments of one imported message, in order. */
+export const ArchiveAttachments = ({ media, api }) =>
+  el(
+    'div',
+    {
+      className: 'remote-media archive-media',
+      'data-archive-media': String(media.items.length),
+    },
+    media.items.map((item, index) => el(Item, { key: index, item, api }))
+  );

package/js/src/web/chat-media.js

@@ -25,6 +25,7 @@ import { useNowSeconds } from './live-clock.js';
 import { useAutoLoad } from './auto-download.js';
 import { LottieSticker } from './lottie-sticker.js';
 import { RemoteAttachments } from './chat-remote-media.js';
+import { ArchiveAttachments } from './chat-archive-media.js';
 import {
   Transcribable,
   TranscribeButton,
@@ -913,6 +914,8 @@ const LiveStreamCard = ({ media }) => {
 const CARDS = {
   // All the attachments of a message at another network (R-Z8).
   remote: RemoteAttachments,
+  // Files kept from an imported archive (R-Z15).
+  archive: ArchiveAttachments,
   'live-stream': LiveStreamCard,
   place: PlaceCard,
   contact: ContactCard,

package/js/src/web/chat-message.js

@@ -292,7 +292,11 @@ const Reactions = ({ reactions, paid, api, chat, msgId }) => {
       ...(reactions ?? []).map((r) =>
         el(
           'span',
-          { key: r.emoticon, className: 'bubble-reaction' },
+          {
+            key: r.emoticon,
+            className: 'bubble-reaction',
+            title: r.by?.length ? r.by.join(', ') : undefined,
+          },
           `${r.emoticon} ${r.count}`
         )
       ),

package/js/src/web/dom.js

@@ -1334,6 +1334,12 @@ const rawApi = {
     return scopedUrl(origin ? new URL(path, origin).toString() : path);
   },
 
+  // An imported archive file's address (R-Z15): `/api/media/<network>/<file>`.
+  archiveMediaUrl: async (path) => {
+    const { origin } = await ensure();
+    return scopedUrl(origin ? new URL(path, origin).toString() : path);
+  },
+
   // Fetch new mail from the saved connection (R-Z5): { ok, imported, … }
   // or { ok: false, error, code }.
   emailPull: async (body = {}) =>

package/js/src/web/locales/en-chat-account.js

@@ -446,4 +446,8 @@ export const enChatAccount = {
     '{members} members read from {networks} networks.',
   'automations.audiences.syncFailed':
     'The members could not be read. Try again.',
+  // Imported archive media (R-Z15).
+  'chat.archive.notImported': 'Media not imported: {name}',
+  'chat.archive.notImportedShort': 'Media not imported',
+  'chat.archive.save': 'Save {name}',
 };

package/js/src/web/locales/hi-chat-account.js

@@ -450,4 +450,8 @@ export const hiChatAccount = {
   'automations.audiences.synced':
     '{networks} नेटवर्क से {members} सदस्य पढ़े गए।',
   'automations.audiences.syncFailed': 'सदस्य नहीं पढ़े जा सके। फिर कोशिश करें।',
+  // Imported archive media (R-Z15).
+  'chat.archive.notImported': 'मीडिया आयात नहीं हुआ: {name}',
+  'chat.archive.notImportedShort': 'मीडिया आयात नहीं हुआ',
+  'chat.archive.save': '{name} सहेजें',
 };

package/js/src/web/locales/ru-chat-account.js

@@ -471,4 +471,8 @@ export const ruChatAccount = {
     'Прочитано участников: {members}, сетей: {networks}.',
   'automations.audiences.syncFailed':
     'Не удалось прочитать участников. Попробуйте ещё раз.',
+  // Imported archive media (R-Z15).
+  'chat.archive.notImported': 'Медиа не импортировано: {name}',
+  'chat.archive.notImportedShort': 'Медиа не импортировано',
+  'chat.archive.save': 'Сохранить {name}',
 };

package/js/src/web/locales/zh-chat-account.js

@@ -410,4 +410,8 @@ export const zhChatAccount = {
   'automations.audiences.synced':
     '已从 {networks} 个网络读取 {members} 名成员。',
   'automations.audiences.syncFailed': '无法读取成员，请重试。',
+  // Imported archive media (R-Z15).
+  'chat.archive.notImported': '媒体未导入：{name}',
+  'chat.archive.notImportedShort': '媒体未导入',
+  'chat.archive.save': '保存 {name}',
 };

package/js/src/web/message-parts.js

@@ -8,6 +8,7 @@
 // code copied — the algorithm is Telegram's own spec).
 
 import { parsePoll, parseTodo } from './poll-parts.js';
+import { archiveItem } from './archive-media-model.js';
 import { quoteRichText, richOf } from './rich-message-model.js';
 import { richServiceWords } from './rich-service-model.js';
 import { inTopicOnly } from './forum-topics.js';
@@ -683,7 +684,12 @@ const parseReactions = (raw, link) =>
         .map((r) => ({ emoticon: r.reaction.emoticon, count: r.count ?? 1 }))
     : (Array.isArray(link?.reactions) ? link.reactions : [])
         .filter((r) => r?.emoji)
-        .map((r) => ({ emoticon: r.emoji, count: r.count ?? 1 }));
+        .map((r) => ({
+          emoticon: r.emoji,
+          count: r.count ?? 1,
+          // Who reacted, when the import kept it (R-Z15).
+          by: Array.isArray(r.by) ? r.by.map(String) : [],
+        }));
 
 // A label for an attachment that is not a file (a place, a contact…).
 const ATTACHMENT_LABELS = {
@@ -706,6 +712,22 @@ const remoteMedia = (list) => ({
   })),
 });
 
+// Imported archives keep their files in media/ (R-Z15): all attachments
+// of such a message are drawn together (./chat-archive-media.js).
+const ARCHIVE_SOURCES = new Set([
+  'facebook',
+  'instagram',
+  'x',
+  'linkedin',
+  'whatsapp',
+]);
+
+const archiveMedia = (list) => ({
+  kind: 'archive',
+  type: list[0].kind ?? 'file',
+  items: list.map(archiveItem),
+});
+
 // One attachment as a link or a file.
 const singleAttachmentMedia = (a) => {
   if (a.kind === 'link' && a.url) {
@@ -734,13 +756,17 @@ const singleAttachmentMedia = (a) => {
  * attachments together as `remote` (all of them), else the first one as a
  * link or a file.
  */
-export const attachmentMedia = (attachments) => {
+export const attachmentMedia = (attachments, source = null) => {
+  const archive = ARCHIVE_SOURCES.has(source);
   const list = (Array.isArray(attachments) ? attachments : []).filter(
-    (a) => a?.kind
+    (a) => a?.kind || (archive && a?.type)
   );
   if (list.length === 0) {
     return null;
   }
+  if (archive) {
+    return archiveMedia(list);
+  }
   return list.length > 1 || REMOTE_KINDS.has(list[0].kind)
     ? remoteMedia(list)
     : singleAttachmentMedia(list[0]);
@@ -751,6 +777,11 @@ export const attachmentMedia = (attachments) => {
 const bodyOf = (link, rich) =>
   typeof link?.body === 'string' && !rich && !link.bodyTree ? link.body : '';
 
+const linkMedia = (link) =>
+  link?.source === 'email'
+    ? null
+    : attachmentMedia(link?.attachments, link?.source);
+
 /**
  * Parse a message link into all display parts.
  *
@@ -761,9 +792,7 @@ const bodyOf = (link, rich) =>
 export const parseMessageParts = (link, msgById = new Map()) => {
   const raw = link?.raw ?? {};
   // A mail lists its attachments itself (chat-email-body.js).
-  const media =
-    parseMedia(raw) ??
-    (link?.source === 'email' ? null : attachmentMedia(link?.attachments));
+  const media = parseMedia(raw) ?? linkMedia(link);
   const rich = richOf(raw);
   const body = bodyOf(link, rich);
   return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "meta-sovereign",
-  "version": "0.49.8",
+  "version": "0.49.10",
   "description": "Personal meta profile sovereign system - fully local, privacy-respecting personal CRM and unified messenger across services",
   "type": "module",
   "main": "js/src/index.js",
@@ -43,6 +43,8 @@
     "test:e2e:browser:webkit": "E2E_BROWSER=webkit node js/tests/e2e-browser-spa.mjs",
     "test:e2e:pages": "node js/tests/e2e-pages.mjs",
     "test:pack": "node js/tests/pack-smoke.mjs",
+    "test:e2e:archive-media": "node js/tests/e2e-archive-media-browser.mjs",
+    "test:e2e:archive-media:webkit": "E2E_BROWSER=webkit node js/tests/e2e-archive-media-browser.mjs",
     "test:e2e:email": "node js/tests/e2e-email-browser.mjs",
     "test:e2e:email:webkit": "E2E_BROWSER=webkit node js/tests/e2e-email-browser.mjs",
     "test:e2e:cv": "node js/tests/e2e-cv-browser.mjs",