meta-sovereign 0.49.8 → 0.49.9

package/js/src/sources/archive/media-store.js

@@ -8,6 +8,7 @@
 import { createHash, randomBytes } from 'node:crypto';
 import { createWriteStream, promises as fs } from 'node:fs';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import { pipeline } from 'node:stream/promises';
 
 const extensionOf = (name) => {
@@ -39,7 +40,7 @@ export const saveMedia = async ({ mediaDir, network, stream, name }) => {
     );
     const sha256 = hash.digest('hex');
     const file = `${sha256}${extensionOf(name)}`;
-    await fs.rename(temp, path.join(dir, file));
+    await renameDurable(temp, path.join(dir, file)); // (R-Z19)
     await fs.chmod(path.join(dir, file), 0o600);
     return { path: `media/${network}/${file}`, sha256, size };
   } catch (error) {

package/js/src/sources/auto-import/sign-ins.js

@@ -28,6 +28,7 @@
 import { promises as fs } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import { execFile } from 'node:child_process';
 import { autoImportRecipes } from './recipes.js';
 
@@ -229,7 +230,7 @@ export const createSignIns = ({
     const file = fileOf(provider);
     const temp = `${file}.${process.pid}.tmp`;
     await fs.writeFile(temp, JSON.stringify(seed), { mode: 0o600 });
-    await fs.rename(temp, file);
+    await renameDurable(temp, file); // (R-Z19)
   };
 
   const access = () => probeFullDiskAccess({ platform, homeDir, env });

package/js/src/sources/auto-import/telegram-creds.js

@@ -32,6 +32,7 @@
 import { promises as fs } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { renameDurable } from '../../storage/durable.js';
 import crypto from 'node:crypto';
 import { timedRequest } from '../http.js';
 
@@ -765,7 +766,7 @@ export const createTelegramCreds = ({
       typeof v === 'bigint' ? v.toString() : v
     );
     await fs.writeFile(temp, text, { mode: 0o600 });
-    await fs.rename(temp, cacheFile);
+    await renameDurable(temp, cacheFile); // (R-Z19)
   };
 
   const verifyAll = (records) =>

package/js/src/sources/telegram-export-run.js

@@ -10,6 +10,7 @@
 
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
+import { renameDurable } from '../storage/durable.js';
 import { rawMessage, floodSeconds } from './telegram-history.js';
 import { bigSize, locationOf, fileLocation } from './telegram-media.js';
 import { failureOf } from './telegram-stickers.js';
@@ -149,7 +150,7 @@ const download = async (r, info, target) => {
         outputFile: part,
       })
     );
-    await fs.rename(part, target);
+    await renameDurable(part, target); // (R-Z19)
   } catch (error) {
     await fs.rm(part, { force: true });
     throw error;

package/js/src/storage/durable.js

@@ -45,6 +45,46 @@ export const fsyncDir = async (dir) => {
   }
 };
 
+const WINDOWS_BUSY = new Set(['EPERM', 'EBUSY', 'EACCES']);
+const defaultSleep = (ms) =>
+  new Promise((resolve) => {
+    globalThis.setTimeout(resolve, ms);
+  });
+
+/**
+ * Rename `from` over `to`, retrying while another handle holds the target
+ * (R-Z19). On Windows a rename fails with EPERM/EBUSY/EACCES while any
+ * process (antivirus, the search indexer, our own snapshot worker) has the
+ * target open, usually for a few milliseconds: retried up to `attempts`
+ * times with a growing pause (about a second in all, like graceful-fs).
+ * Elsewhere rename is atomic and those codes are real errors, so only
+ * EBUSY is retried. After the last attempt the error is thrown and the
+ * target is untouched. `rename`, `platform` and `sleep` are injectable.
+ */
+export const renameDurable = async (
+  from,
+  to,
+  {
+    rename = fs.rename,
+    platform = process.platform,
+    sleep = defaultSleep,
+    attempts = 10,
+    delayMs = 20,
+  } = {}
+) => {
+  const codes = platform === 'win32' ? WINDOWS_BUSY : new Set(['EBUSY']);
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await rename(from, to);
+    } catch (error) {
+      if (attempt >= attempts || !codes.has(error?.code)) {
+        throw error;
+      }
+      await sleep(delayMs * attempt);
+    }
+  }
+};
+
 /** Create `dir` (and parents) private to the user. */
 export const ensurePrivateDir = (dir) =>
   fs.mkdir(dir, { recursive: true, mode: DIR_MODE });
@@ -74,7 +114,7 @@ export const tighten = async (target, mode = FILE_MODE) => {
 export const writeFileDurable = async (
   file,
   data,
-  { utime, tag = '' } = {}
+  { utime, tag = '', rename } = {}
 ) => {
   const dir = path.dirname(file);
   await ensurePrivateDir(dir);
@@ -90,7 +130,7 @@ export const writeFileDurable = async (
     } finally {
       await handle.close();
     }
-    await fs.rename(tmp, file);
+    await renameDurable(tmp, file, { rename });
   } catch (error) {
     // A full disk: the half-written temp file goes at once, not at the
     // next start (R-Z14).
@@ -116,7 +156,7 @@ export const writeStreamDurable = async (file, ...streams) => {
       createWriteStream(tmp, { flags: 'w', mode: FILE_MODE })
     );
     await fsyncFile(tmp);
-    await fs.rename(tmp, file);
+    await renameDurable(tmp, file);
   } catch (error) {
     await fs.unlink(tmp).catch(() => {});
     throw error;

package/js/src/storage/lino-store.js

@@ -28,7 +28,13 @@ import { createInterface } from 'node:readline';
 import { decodeLine, encodeLine } from 'lino-objects-codec';
 import { createMemoryStore } from './universal.js';
 import { parseLino } from './lino.js';
-import { FILE_MODE, fsyncDir, fsyncFile, tighten } from './durable.js';
+import {
+  FILE_MODE,
+  fsyncDir,
+  fsyncFile,
+  renameDurable,
+  tighten,
+} from './durable.js';
 import { enoughSpace, skippedEvent } from './disk-space.js';
 import { createWriteQueue, storeError, writeFailed } from './write-queue.js';
 
@@ -380,7 +386,7 @@ export const readLinoFile = async (filePath) => {
 // Write every link to a temp file in chunks, flush it to disk, swap it
 // in, and flush the directory: after a power cut the file is the old log
 // or the whole new one, never an empty or half-written one (R-Z2).
-const rewriteFile = async (filePath, links) => {
+const rewriteFile = async (filePath, links, rename) => {
   await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
   const tmp = `${filePath}.${process.pid}.tmp`;
   try {
@@ -393,7 +399,7 @@ const rewriteFile = async (filePath, links) => {
     } finally {
       await handle.close();
     }
-    await fs.rename(tmp, filePath);
+    await renameDurable(tmp, filePath, { rename });
   } catch (error) {
     // The half-written copy goes at once: on a full disk it is what
     // keeps the disk full (R-Z14).
@@ -461,6 +467,9 @@ const SYNC_MS = 200;
 
 // A compaction skipped for lack of space is asked again after this long (R-Z14).
 const COMPACT_RETRY_MS = 10 * 60_000;
+// A compaction that failed (a busy file on Windows) leaves the old log intact
+// and is asked again after this, not on every batch (R-Z19).
+const COMPACT_FAILED_RETRY_MS = 30_000;
 
 // Flushes appends to disk together, `syncMs` after the first (R-Z2).
 const createSyncer = (filePath, syncMs, enqueue) => {
@@ -506,7 +515,15 @@ const compactionFailed = (onEvent) => (error) =>
 // The log rewritten with one line per live link, after the unreadable
 // lines were set aside (R-Z2); skipped when the disk has no room for the
 // copy (R-Z14). `{ skipped }`.
-const compactLog = async ({ filePath, bad, mem, onCorrupt, onEvent, free }) => {
+const compactLog = async ({
+  filePath,
+  bad,
+  mem,
+  onCorrupt,
+  onEvent,
+  free,
+  rename,
+}) => {
   if (bad.overflow) {
     return { skipped: false }; // more unreadable lines than can be kept
   }
@@ -522,7 +539,7 @@ const compactLog = async ({ filePath, bad, mem, onCorrupt, onEvent, free }) => {
     bad.lines = [];
     bad.count = 0;
   }
-  await rewriteFile(filePath, await mem.query());
+  await rewriteFile(filePath, await mem.query(), rename);
   return { skipped: false };
 };
 
@@ -588,6 +605,14 @@ const createHold = () => {
   };
 };
 
+// A compaction skipped for lack of space, as an error for the write queue.
+const throwingWhenSkipped = (compact) => async () => {
+  const done = await compact();
+  if (done?.skipped) {
+    throw storeError('ENOSPC', 'not enough free space to rewrite the log');
+  }
+};
+
 const readOnlyError = () =>
   Object.assign(new Error('the store was opened read-only'), {
     code: 'STORE_READ_ONLY',
@@ -607,6 +632,7 @@ export const createLinoTextStore = async (
     // (tests inject an append handle that fails).
     queue: queueOptions = {},
     measureFree,
+    rename, // injectable fs.rename (R-Z19)
   } = {}
 ) => {
   const mem = createMemoryStore();
@@ -642,6 +668,7 @@ export const createLinoTextStore = async (
       onCorrupt,
       onEvent,
       free: measureFree,
+      rename,
     });
     if (done.skipped) {
       compactAfterAt = Date.now() + COMPACT_RETRY_MS;
@@ -676,16 +703,9 @@ export const createLinoTextStore = async (
     attempt: () => (rewrite ? compactOrThrow() : wq.write()),
     onRecovered: syncer.soon,
   });
-  const compactOrThrow = async () => {
-    const done = await compact();
-    if (done?.skipped) {
-      throw storeError('ENOSPC', 'not enough free space to rewrite the log');
-    }
-  };
+  const compactOrThrow = throwingWhenSkipped(compact);
   const hold = createHold();
   const maintain = async () => {
-    // The batch is on disk by now; a failed compaction costs nothing but
-    // the rewrite, so it never fails the writes (R-Z14).
     try {
       if (
         !hold.active() &&
@@ -695,6 +715,7 @@ export const createLinoTextStore = async (
         await compact();
       }
     } catch (error) {
+      compactAfterAt = Date.now() + COMPACT_FAILED_RETRY_MS;
       compactionFailed(onEvent)(error);
     }
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "meta-sovereign",
-  "version": "0.49.8",
+  "version": "0.49.9",
   "description": "Personal meta profile sovereign system - fully local, privacy-respecting personal CRM and unified messenger across services",
   "type": "module",
   "main": "js/src/index.js",