@procedures/fs 0.27.4 → 0.27.5

package/src/files/atomic.ts

@@ -1,17 +1,30 @@
 /**
- * The atomic write of a file (roadmap O51).
+ * The atomic write of a file (roadmap O51 and O52).
  *
  * `writeFileAtomic()` writes the data to a new temporary file in the folder of the target. Then it
  * renames the temporary file over the target. Thus a reader sees the old file or the whole new
- * file, also after a crash.
+ * file, also after a crash. `writeFileAtomicSync()` does the same steps without a promise.
  */
 
 import { randomBytes } from "node:crypto";
+import {
+  chmodSync,
+  closeSync,
+  fchmodSync,
+  fsyncSync,
+  openSync,
+  realpathSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { chmod, open, realpath, rename, stat, unlink, type FileHandle } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
 import { setTimeout as sleep } from "node:timers/promises";
+import { perform, runAsync, runSync } from "./steps.js";
 
-/** The options of `writeFileAtomic()`. */
+/** The options of `writeFileAtomic()` and `writeFileAtomicSync()`. */
 export interface WriteFileAtomicOptions {
   /** The encoding of a string `data`. The default is `utf8`. */
   encoding?: BufferEncoding | undefined;
@@ -27,6 +40,15 @@ export interface WriteFileAtomicOptions {
    * is 2000 on Windows and 0 on the other platforms.
    */
   renameRetryMs?: number | undefined;
+  /**
+   * The function that gives the name of the temporary file. It gets the absolute path of the target,
+   * after the function follows a symlink.
+   *
+   * The temporary file is in the folder of the target, so a name with `/` or `\` is an error. The
+   * default name is `.<name>.<16 hex digits>.tmp`, where `<name>` is the first 32 characters of the
+   * name of the target.
+   */
+  tempName?: ((target: string) => string) | undefined;
 }
 
 /** On Windows, an antivirus or an indexer that holds the target open gives these errors for a short time. */
@@ -35,6 +57,30 @@ const LOCKED = new Set(["EPERM", "EACCES", "EBUSY"]);
 /** A file system that cannot flush a folder gives these errors. The write is complete without the flush. */
 const NO_FOLDER_SYNC = new Set(["EINVAL", "ENOTSUP", "EISDIR", "EBADF", "EPERM", "EACCES"]);
 
+/** The permission bits that let the group and the others read a file. */
+const OTHERS_READ = 0o044;
+
+/** A file that the steps open: a `FileHandle` for the async form, a file descriptor for the sync form. */
+type Handle = FileHandle | number;
+
+/** One operation of the file system that the steps of a write need. */
+type FileOp =
+  | { readonly op: "realpath"; readonly path: string }
+  | { readonly op: "mode"; readonly path: string }
+  | { readonly op: "create"; readonly path: string; readonly mode: number }
+  | { readonly op: "openFolder"; readonly path: string }
+  | { readonly op: "write"; readonly handle: Handle; readonly data: string | Uint8Array; readonly encoding: BufferEncoding }
+  | { readonly op: "fchmod"; readonly handle: Handle; readonly mode: number }
+  | { readonly op: "fsync"; readonly handle: Handle }
+  | { readonly op: "close"; readonly handle: Handle }
+  | { readonly op: "rename"; readonly from: string; readonly to: string }
+  | { readonly op: "unlink"; readonly path: string }
+  | { readonly op: "chmod"; readonly path: string; readonly mode: number }
+  | { readonly op: "sleep"; readonly ms: number };
+
+/** The steps of a write. Each step yields one operation, and the runner gives back its result. */
+type Steps<T> = Generator<FileOp, T, unknown>;
+
 /** True on Windows. The function reads the platform at each call, so a test can change it. */
 function isWindows(): boolean {
   return process.platform === "win32";
@@ -52,10 +98,10 @@ function isMissing(error: unknown): boolean {
 }
 
 /** The absolute path of the file to replace. The function follows a symlink, so the link stays a link. */
-async function targetOf(path: string): Promise<string> {
+function* targetOf(path: string): Steps<string> {
   const absolute = resolve(path);
   try {
-    return await realpath(absolute);
+    return yield* perform<FileOp, string>({ op: "realpath", path: absolute });
   } catch (error) {
     if (isMissing(error)) return absolute;
     throw error;
@@ -63,53 +109,75 @@ async function targetOf(path: string): Promise<string> {
 }
 
 /** The permission bits of the target, or undefined when the target does not exist. */
-async function modeOf(target: string): Promise<number | undefined> {
+function* modeOf(target: string): Steps<number | undefined> {
   try {
-    return (await stat(target)).mode & 0o7777;
+    return yield* perform<FileOp, number>({ op: "mode", path: target });
   } catch (error) {
     if (isMissing(error)) return undefined;
     throw error;
   }
 }
 
+/** The `ERR_INVALID_ARG_VALUE` error of a name of a temporary file that is not in the folder of the target. */
+function badTempName(name: unknown): TypeError {
+  const error = new TypeError(
+    `The option tempName gave ${JSON.stringify(name)}. Give the name of a new file in the folder of the target, without "/" or "\\".`,
+  );
+  return Object.assign(error, { code: "ERR_INVALID_ARG_VALUE" });
+}
+
+/**
+ * A new path next to the target. The default name starts with a dot and ends in `.tmp`. Thus a
+ * reader that looks for the extension of the target does not see it. The name keeps 32 characters
+ * of the target, so it does not become too long.
+ */
+function tempPathOf(target: string, tempName: WriteFileAtomicOptions["tempName"]): string {
+  const own = basename(target);
+  if (!tempName) return join(dirname(target), `.${[...own].slice(0, 32).join("")}.${randomBytes(8).toString("hex")}.tmp`);
+  const name: unknown = tempName(target);
+  if (typeof name !== "string" || name === "" || name === "." || name === ".." || name === own || /[\\/]/.test(name)) {
+    throw badTempName(name);
+  }
+  return join(dirname(target), name);
+}
+
 /**
- * A new path next to the target. The name starts with a dot and ends in `.tmp`, so a reader that
- * looks for the extension of the target does not see it. The name keeps 32 characters of the
- * target, so it does not become too long.
+ * The permissions of the new temporary file. When the final file hides its data from the group and
+ * the others, the temporary file gets `0o600` from the start. Then no other user can read it.
  */
-function tempPathOf(target: string): string {
-  const name = [...basename(target)].slice(0, 32).join("");
-  return join(dirname(target), `.${name}.${randomBytes(8).toString("hex")}.tmp`);
+function createModeOf(mode: number | undefined): number {
+  if (mode === undefined) return 0o666;
+  return (mode & OTHERS_READ) === 0 ? 0o600 : mode;
 }
 
 /**
  * The rename, tried again with a longer wait each time until `retryMs` is over. A random part of
  * each wait keeps two writers of one target out of step.
  */
-async function renameWithRetry(from: string, to: string, retryMs: number): Promise<void> {
+function* renameWithRetry(from: string, to: string, retryMs: number): Steps<void> {
   const deadline = Date.now() + retryMs;
   for (let step = 10; ; step = Math.min(step * 2, 100)) {
     try {
-      await rename(from, to);
+      yield { op: "rename", from, to };
       return;
     } catch (error) {
       const delay = Math.round(step * (0.5 + Math.random()));
       if (!LOCKED.has(codeOf(error)) || Date.now() + delay > deadline) throw error;
-      await sleep(delay);
+      yield { op: "sleep", ms: delay };
     }
   }
 }
 
 /** The removal of the temporary file after a failure. The original error is more important than an error here. */
-async function removeTemp(temp: string): Promise<void> {
+function* removeTemp(temp: string): Steps<void> {
   try {
-    await unlink(temp);
+    yield { op: "unlink", path: temp };
   } catch (error) {
     if (codeOf(error) !== "EPERM") return;
     // Windows does not remove a read-only file, so the function clears the flag and tries again
     try {
-      await chmod(temp, 0o666);
-      await unlink(temp);
+      yield { op: "chmod", path: temp, mode: 0o666 };
+      yield { op: "unlink", path: temp };
     } catch {
       // The temporary file stays. The error of the write is the error that the caller gets.
     }
@@ -117,63 +185,140 @@ async function removeTemp(temp: string): Promise<void> {
 }
 
 /** The flush of the folder, so that the rename is on the disk. Windows cannot open a folder for it. */
-async function syncFolder(folder: string): Promise<void> {
+function* syncFolder(folder: string): Steps<void> {
   if (isWindows()) return;
-  let handle: FileHandle | undefined;
+  let handle: Handle | undefined;
   try {
-    handle = await open(folder, "r");
-    await handle.sync();
+    handle = yield* perform<FileOp, Handle>({ op: "openFolder", path: folder });
+    yield { op: "fsync", handle };
   } catch (error) {
     if (!NO_FOLDER_SYNC.has(codeOf(error))) throw error;
   } finally {
-    await handle?.close();
+    if (handle !== undefined) yield { op: "close", handle };
   }
 }
 
-/**
- * This function writes `data` to `path` in one step for each reader. A reader sees the old file or
- * the whole new file. It does not see a part of a file at any time.
- *
- * - The function writes a new temporary file in the folder of the target and flushes it to the disk.
- * - Then it renames the temporary file over the target and flushes the folder. Windows gets no flush of the folder.
- * - On Windows, an open handle of the target blocks the rename. Then the function tries the rename again for `renameRetryMs`.
- * - After a failure, the function removes the temporary file and throws the error.
- * - The file keeps the permissions of the target, unless `options.mode` is set. It does not keep the owner.
- * - When `path` is a symlink, the function replaces the target of the link.
- *
- * Two writes at the same time do not mix: the file gets the data of one of them. The folder must exist.
- */
-export async function writeFileAtomic(
-  path: string,
-  data: string | Uint8Array,
-  options: WriteFileAtomicOptions = {},
-): Promise<void> {
-  const target = await targetOf(path);
-  const mode = options.mode ?? (await modeOf(target));
+/** The steps of `writeFileAtomic()` and `writeFileAtomicSync()`. */
+function* writeSteps(path: string, data: string | Uint8Array, options: WriteFileAtomicOptions): Steps<void> {
+  const target = yield* targetOf(path);
+  const mode = options.mode ?? (yield* modeOf(target));
   const flush = options.fsync ?? true;
-  const temp = tempPathOf(target);
+  const temp = tempPathOf(target, options.tempName);
   let created = false;
-  let handle: FileHandle | undefined;
+  let handle: Handle | undefined;
   try {
-    handle = await open(temp, "wx", mode ?? 0o666);
+    handle = yield* perform<FileOp, Handle>({ op: "create", path: temp, mode: createModeOf(mode) });
     created = true;
-    await handle.writeFile(data, { encoding: options.encoding ?? "utf8" });
-    // The umask changes the mode of open(), so the function sets the mode again
-    if (mode !== undefined) await handle.chmod(mode);
-    if (flush) await handle.sync();
-    await handle.close();
+    yield { op: "write", handle, data, encoding: options.encoding ?? "utf8" };
+    // The umask changes the mode of the open, and a secret file starts at 0o600, so the function sets the final mode
+    if (mode !== undefined) yield { op: "fchmod", handle, mode };
+    if (flush) yield { op: "fsync", handle };
+    yield { op: "close", handle };
     handle = undefined;
-    await renameWithRetry(temp, target, options.renameRetryMs ?? (isWindows() ? 2000 : 0));
+    yield* renameWithRetry(temp, target, options.renameRetryMs ?? (isWindows() ? 2000 : 0));
   } catch (error) {
-    if (handle) {
+    if (handle !== undefined) {
       try {
-        await handle.close();
+        yield { op: "close", handle };
       } catch {
         // The error of the write is the error that the caller gets
       }
     }
-    if (created) await removeTemp(temp);
+    if (created) yield* removeTemp(temp);
     throw error;
   }
-  if (flush) await syncFolder(dirname(target));
+  if (flush) yield* syncFolder(dirname(target));
+}
+
+/** The async operations of the file system, with `node:fs/promises`. */
+async function doAsync(op: FileOp): Promise<unknown> {
+  switch (op.op) {
+    case "realpath":
+      return realpath(op.path);
+    case "mode":
+      return (await stat(op.path)).mode & 0o7777;
+    case "create":
+      return open(op.path, "wx", op.mode);
+    case "openFolder":
+      return open(op.path, "r");
+    case "write":
+      return (op.handle as FileHandle).writeFile(op.data, { encoding: op.encoding });
+    case "fchmod":
+      return (op.handle as FileHandle).chmod(op.mode);
+    case "fsync":
+      return (op.handle as FileHandle).sync();
+    case "close":
+      return (op.handle as FileHandle).close();
+    case "rename":
+      return rename(op.from, op.to);
+    case "unlink":
+      return unlink(op.path);
+    case "chmod":
+      return chmod(op.path, op.mode);
+    case "sleep":
+      return sleep(op.ms);
+  }
+}
+
+let sleeper: Int32Array | undefined;
+
+/** The sync operations of the file system, with `node:fs`. A wait blocks the thread with `Atomics.wait()`. */
+function doSync(op: FileOp): unknown {
+  switch (op.op) {
+    case "realpath":
+      return realpathSync.native(op.path);
+    case "mode":
+      return statSync(op.path).mode & 0o7777;
+    case "create":
+      return openSync(op.path, "wx", op.mode);
+    case "openFolder":
+      return openSync(op.path, "r");
+    case "write":
+      return writeFileSync(op.handle as number, op.data, { encoding: op.encoding });
+    case "fchmod":
+      return fchmodSync(op.handle as number, op.mode);
+    case "fsync":
+      return fsyncSync(op.handle as number);
+    case "close":
+      return closeSync(op.handle as number);
+    case "rename":
+      return renameSync(op.from, op.to);
+    case "unlink":
+      return unlinkSync(op.path);
+    case "chmod":
+      return chmodSync(op.path, op.mode);
+    case "sleep":
+      sleeper ??= new Int32Array(new SharedArrayBuffer(4));
+      return Atomics.wait(sleeper, 0, 0, op.ms);
+  }
+}
+
+/**
+ * This function writes `data` to `path` in one step for each reader. A reader sees the old file or
+ * the whole new file. It does not see a part of a file at any time.
+ *
+ * - The function writes a new temporary file in the folder of the target and flushes it to the disk.
+ * - Then it renames the temporary file over the target and flushes the folder. Windows gets no flush of the folder.
+ * - On Windows, an open handle of the target blocks the rename. Then the function tries the rename again for `renameRetryMs`.
+ * - After a failure, the function removes the temporary file and throws the error.
+ * - The file keeps the permissions of the target, unless `options.mode` is set. It does not keep the owner.
+ * - When the group and the others cannot read the final file, the temporary file gets `0o600` before the write of the data.
+ * - When `path` is a symlink, the function replaces the target of the link.
+ *
+ * Two writes at the same time do not mix: the file gets the data of one of them. The folder must exist.
+ * The name of the temporary file is `.<name>.<16 hex digits>.tmp`, unless `options.tempName` gives a name.
+ */
+export function writeFileAtomic(path: string, data: string | Uint8Array, options: WriteFileAtomicOptions = {}): Promise<void> {
+  return runAsync(writeSteps(path, data, options), doAsync);
+}
+
+/**
+ * This function is the sync form of `writeFileAtomic()`. It does the same steps, with the same
+ * options and the same errors.
+ *
+ * - It blocks the thread until the file is in place.
+ * - On Windows, a locked target can block the thread for `renameRetryMs` (2000 by default) while the function tries the rename again.
+ */
+export function writeFileAtomicSync(path: string, data: string | Uint8Array, options: WriteFileAtomicOptions = {}): void {
+  runSync(writeSteps(path, data, options), doSync);
 }

package/src/files/edit.ts

@@ -62,7 +62,8 @@ async function readOrUndefined(path: string, encoding: BufferEncoding): Promise<
  * - A transform that gives the same text, or `undefined`, writes nothing.
  * - Two paths of the same file share the lock, also when the case of the letters is not the same on Windows.
  *
- * The lock is in the process. It does not stop another process. The options of `writeFileAtomic()` apply to the write.
+ * The lock is in the process. It does not stop another process. The options of `writeFileAtomic()` apply to the write,
+ * also `tempName` and the `0o600` temporary file of a secret file.
  */
 export function editFile(
   path: string,

package/src/files/index.ts

@@ -1,15 +1,22 @@
 /**
- * The file helpers of the package (roadmap O51): the `./files` entry. They are functions, not
- * procedures, and an import of this entry registers nothing.
+ * The file helpers of the package (roadmap O51 and O52): the `./files` entry. They are functions,
+ * not procedures, and an import of this entry registers nothing.
  *
- * - `writeFileAtomic()` writes a file in one step for each reader.
- * - `healNdjsonTail()` repairs the end of an append-only NDJSON file after a crash.
+ * - `writeFileAtomic()` and `writeFileAtomicSync()` write a file in one step for each reader.
+ * - `healNdjsonTail()` and `healNdjsonTailSync()` repair the end of an append-only NDJSON file after a crash.
  * - `KeyedLock` and `withLock()` let one caller at a time hold a key.
  * - `editFile()` changes a file, one edit at a time for each path.
  */
 
-export { writeFileAtomic, type WriteFileAtomicOptions } from "./atomic.js";
-export { healNdjsonTail, type HealNdjsonTailOptions, type NdjsonTailAction, type NdjsonTailResult } from "./ndjson-tail.js";
+export { writeFileAtomic, writeFileAtomicSync, type WriteFileAtomicOptions } from "./atomic.js";
+export {
+  healNdjsonTail,
+  healNdjsonTailSync,
+  NDJSON_LINE_TOO_LONG,
+  type HealNdjsonTailOptions,
+  type NdjsonTailAction,
+  type NdjsonTailResult,
+} from "./ndjson-tail.js";
 export {
   KeyedLock,
   withLock,