@napi-rs/cli 3.10.1 → 3.10.3

package/src/utils/__tests__/misc.spec.ts

@@ -1,11 +1,16 @@
+import { createHash } from 'node:crypto'
 import { existsSync, type BigIntStats } from 'node:fs'
 import {
+  appendFile,
+  chmod,
   lstat,
   mkdtemp,
   readdir,
   readFile,
   rename,
   rm,
+  stat,
+  utimes,
   writeFile,
 } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
@@ -15,6 +20,7 @@ import ava, { type TestFn } from 'ava'
 
 import {
   retireFailedSnapshotLeftover,
+  snapshotFileSystemTransactionInput,
   snapshotLeftoverIsTransactionOwned,
   statIdentitiesMatch,
   updatePackageJson,
@@ -179,3 +185,336 @@ test('retireFailedSnapshotLeftover restores a successor swapped in during the ra
   t.is(await readFile(destination, 'utf8'), 'successor content')
   t.deepEqual(await readdir(t.context.tmpDir), ['leftover.tmp'])
 })
+
+// A snapshot re-verifies its source after the copy so the recorded image is
+// provably consistent. The check used to fail the whole transaction on any
+// difference, including a metadata-only re-stamp of an inode the cli had just
+// written itself — the FreeBSD release-build failure in rolldown/rolldown#10268
+// — and its message named none of the eight conditions it stood for.
+//
+// `onAfterCopy` is the seam these tests drive: it runs after each copy pass and
+// before the post-copy stats, so drift is injected deterministically rather than
+// by racing a timer against a large copy.
+//
+// Every mutation below has to be deterministic on the Windows lanes too
+// (`.github/workflows/test-release.yaml` runs `yarn test:cli` on both
+// `windows-latest` and `windows-11-arm`), which rules out two tempting ones:
+//
+//   * an identical-mode `chmod` to move `ctime` alone. On Windows that is an
+//     attribute no-op and nothing documents it advancing the NTFS ChangeTime
+//     that libuv reports as `ctimeNs`.
+//   * asserting a full POSIX mode. Node documents that on Windows "only the
+//     write permission can be changed, and the distinction among the
+//     permissions of group, owner, or others is not implemented", so a mode
+//     round-trips as read-only or writable and nothing else.
+//
+// So timestamp drift is forced with `utimes`, whose `mtime` write is defined on
+// every supported platform, and mode drift toggles only the owner write bit and
+// asserts against the mode actually observed afterwards.
+const snapshotSourceBytes = Buffer.from('snapshot source contents')
+const snapshotSourceHash = createHash('sha256')
+  .update(snapshotSourceBytes)
+  .digest('hex')
+
+/**
+ * An in-place rewrite of exactly the same length as {@link snapshotSourceBytes}.
+ * Every field the snapshot compares — `dev`, `ino`, `size`, `bytesRead`, `mode`
+ * and, once the timestamps are restamped, `mtimeNs` — survives it untouched, so
+ * only the content moves. That is the shape a coarse or coalesced filesystem
+ * clock hides, and the only thing that can catch it is the hash.
+ */
+function rewrittenSourceBytes(revision: number) {
+  const bytes = Buffer.from(snapshotSourceBytes)
+  bytes.writeUInt8(0x30 + (revision % 10), bytes.length - 1)
+  return bytes
+}
+
+function sourceHashOf(bytes: Buffer) {
+  return createHash('sha256').update(bytes).digest('hex')
+}
+
+// Two of the mutations below have no Windows equivalent — see the comment on
+// each test for which Windows behavior rules it out.
+const posixTest = process.platform === 'win32' ? test.skip : test
+
+async function writeSnapshotSource(tmpDir: string) {
+  const source = join(tmpDir, 'source.node')
+  await writeFile(source, snapshotSourceBytes)
+  return { destination: join(tmpDir, 'snapshot.input'), source }
+}
+
+async function sourceMode(source: string) {
+  return Number((await stat(source, { bigint: true })).mode & 0o7777n)
+}
+
+/**
+ * Move `mtime` to a fixed whole second, so the drift is the same on a
+ * filesystem with nanosecond stamps and on one with a coarser clock.
+ */
+async function restampSource(source: string, second: number) {
+  const when = new Date(1_700_000_000_000 + second * 1_000)
+  await utimes(source, when, when)
+}
+
+function snapshotInput(
+  source: string,
+  destination: string,
+  onAfterCopy: (attempt: number) => Promise<void>,
+) {
+  return snapshotFileSystemTransactionInput(
+    source,
+    destination,
+    undefined,
+    undefined,
+    0o600,
+    true,
+    undefined,
+    undefined,
+    onAfterCopy,
+  )
+}
+
+// The legitimate FreeBSD case, and the other side of the content-hash rule
+// below: a re-stamp that leaves the bytes alone is accepted on the very next
+// attempt, because that attempt reproduces the hash of the one before it.
+test('snapshot retries a source whose timestamps were re-stamped mid-copy', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const mode = await sourceMode(source)
+  const attempts: number[] = []
+
+  const state = await snapshotInput(source, destination, async (attempt) => {
+    attempts.push(attempt)
+    if (attempt === 1) {
+      // Not one byte of content changes: only the inode's timestamps move.
+      await restampSource(source, 1)
+    }
+  })
+
+  t.deepEqual(attempts, [1, 2])
+  t.is(state.hash, snapshotSourceHash)
+  t.is(state.mode, mode)
+  t.deepEqual(await readFile(destination), snapshotSourceBytes)
+})
+
+posixTest(
+  'snapshot retries a source whose ctime alone was re-stamped mid-copy',
+  async (t) => {
+    const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+    const mode = await sourceMode(source)
+    const attempts: number[] = []
+
+    const state = await snapshotInput(source, destination, async (attempt) => {
+      attempts.push(attempt)
+      if (attempt === 1) {
+        // Re-applying the same mode is the narrowest possible re-stamp: size,
+        // mtime and mode all hold still and only ctime moves. This is the
+        // FreeBSD symptom in its purest form.
+        await chmod(source, mode)
+      }
+    })
+
+    t.deepEqual(attempts, [1, 2])
+    t.is(state.hash, snapshotSourceHash)
+    t.is(state.mode, mode)
+    t.deepEqual(await readFile(destination), snapshotSourceBytes)
+  },
+)
+
+test('snapshot retries mtime drift until the source settles', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const attempts: number[] = []
+
+  const state = await snapshotInput(source, destination, async (attempt) => {
+    attempts.push(attempt)
+    if (attempt < 3) {
+      await restampSource(source, attempt)
+    }
+  })
+
+  // The third attempt is the last the bound allows, and it is clean.
+  t.deepEqual(attempts, [1, 2, 3])
+  t.is(state.hash, snapshotSourceHash)
+  t.deepEqual(await readFile(destination), snapshotSourceBytes)
+})
+
+test('snapshot records the settled mode after a mode re-stamp', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  // Start read-only and hand back the write bit mid-copy. That is the only
+  // mode transition Windows can represent, and it leaves the file writable so
+  // the fixture teardown never meets a read-only inode.
+  await chmod(source, 0o444)
+  const readOnlyMode = await sourceMode(source)
+  const attempts: number[] = []
+
+  const state = await snapshotInput(source, destination, async (attempt) => {
+    attempts.push(attempt)
+    if (attempt === 1) {
+      await chmod(source, 0o644)
+    }
+  })
+
+  // Assert against the mode the platform actually reports, never a literal:
+  // POSIX settles on 0o644 and Windows on 0o666.
+  const writableMode = await sourceMode(source)
+  t.not(readOnlyMode, writableMode, 'the chmod must produce real mode drift')
+  t.deepEqual(attempts, [1, 2])
+  t.is(state.mode, writableMode)
+  t.is(state.hash, snapshotSourceHash)
+})
+
+test('snapshot still fails on a size change and names the field', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const attempts: number[] = []
+
+  const error = await t.throwsAsync(
+    snapshotInput(source, destination, async (attempt) => {
+      attempts.push(attempt)
+      if (attempt === 1) {
+        await appendFile(source, 'appended')
+      }
+    }),
+  )
+
+  // A hard field is fatal on the first observation: no retry is attempted.
+  t.deepEqual(attempts, [1])
+  t.true(
+    error?.message.startsWith(
+      `Filesystem transaction source changed while it was snapshotted: ${source} (`,
+    ),
+  )
+  t.true(
+    error?.message.includes(
+      `size ${snapshotSourceBytes.length} -> ${snapshotSourceBytes.length + 8}`,
+    ),
+    error?.message,
+  )
+  t.true(
+    error?.message.includes(
+      `bytesRead ${snapshotSourceBytes.length} != size ${snapshotSourceBytes.length + 8}`,
+    ),
+    error?.message,
+  )
+  t.false(existsSync(destination))
+})
+
+// Swapping a successor onto a path the snapshot still holds open is POSIX-only:
+// Windows refuses to replace a file with a live handle, and the rename itself
+// fails before the assertion is reached with
+//   EPERM: operation not permitted, rename '...\\successor.node' -> '...\\source.node'
+// The size-change test above already covers a hard field on the Windows lanes.
+posixTest(
+  'snapshot still fails when the path is replaced and names the identity',
+  async (t) => {
+    const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+    const successor = join(t.context.tmpDir, 'successor.node')
+    await writeFile(successor, snapshotSourceBytes)
+    const before = await lstat(source, { bigint: true })
+    const after = await lstat(successor, { bigint: true })
+
+    const error = await t.throwsAsync(
+      snapshotInput(source, destination, async (attempt) => {
+        if (attempt === 1) {
+          await rename(successor, source)
+        }
+      }),
+    )
+
+    t.true(
+      error?.message.includes(
+        `path identity ${before.dev}/${before.ino} -> ${after.dev}/${after.ino}`,
+      ),
+      error?.message,
+    )
+    t.false(existsSync(destination))
+  },
+)
+
+test('snapshot gives up after the attempt bound and names the drifted fields', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const attempts: number[] = []
+
+  const error = await t.throwsAsync(
+    snapshotInput(source, destination, async (attempt) => {
+      attempts.push(attempt)
+      // Never settles: every attempt observes a different mtime.
+      await restampSource(source, attempt)
+    }),
+  )
+
+  t.deepEqual(attempts, [1, 2, 3])
+  t.true(
+    error?.message.startsWith(
+      `Filesystem transaction source changed while it was snapshotted: ${source} (`,
+    ),
+  )
+  t.regex(error?.message ?? '', /mtimeNs \d+ -> \d+/)
+  t.true(
+    error?.message.endsWith('still drifting after 3 snapshot attempts)'),
+    error?.message,
+  )
+  t.false(existsSync(destination))
+})
+
+// Codex review of napi-rs/napi-rs#3530: the retry rebases its baseline purely on
+// metadata, so a writer rewriting the file in place at the same length inside
+// one timestamp tick could hand back a mixed copy that every stat field calls
+// settled. Two attempts now have to agree on the hash before one is accepted.
+test('snapshot re-copies until two attempts agree on the source content', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const mode = await sourceMode(source)
+  const settled = rewrittenSourceBytes(1)
+  const attempts: number[] = []
+
+  const state = await snapshotInput(source, destination, async (attempt) => {
+    attempts.push(attempt)
+    if (attempt === 1) {
+      // Same length, different bytes, and the timestamps pinned to a fixed
+      // second so the second attempt sees metadata that looks perfectly
+      // settled. Only the content betrays the writer.
+      await writeFile(source, settled)
+      await restampSource(source, 1)
+    }
+  })
+
+  // The second attempt copies the settled bytes but cannot know they are
+  // settled — its hash is the first one that differs. Only the third attempt,
+  // which reproduces it, is accepted.
+  t.deepEqual(attempts, [1, 2, 3])
+  t.is(state.hash, sourceHashOf(settled))
+  t.is(state.mode, mode)
+  t.deepEqual(await readFile(destination), settled)
+})
+
+test('snapshot gives up when the source content never settles and names the hash', async (t) => {
+  const { destination, source } = await writeSnapshotSource(t.context.tmpDir)
+  const attempts: number[] = []
+  const copied: Buffer[] = [snapshotSourceBytes]
+
+  const error = await t.throwsAsync(
+    snapshotInput(source, destination, async (attempt) => {
+      attempts.push(attempt)
+      const next = rewrittenSourceBytes(attempt)
+      copied.push(next)
+      await writeFile(source, next)
+      await restampSource(source, attempt)
+    }),
+  )
+
+  t.deepEqual(attempts, [1, 2, 3])
+  t.true(
+    error?.message.startsWith(
+      `Filesystem transaction source changed while it was snapshotted: ${source} (`,
+    ),
+  )
+  t.true(
+    error?.message.includes(
+      `contentHash ${sourceHashOf(copied[1])} -> ${sourceHashOf(copied[2])}`,
+    ),
+    error?.message,
+  )
+  t.true(
+    error?.message.endsWith('still drifting after 3 snapshot attempts)'),
+    error?.message,
+  )
+  t.false(existsSync(destination))
+})

package/src/utils/misc.ts

@@ -4353,7 +4353,83 @@ export async function retireFailedSnapshotLeftover(
   return { outcome: 'kept' }
 }
 
-async function snapshotFileSystemTransactionInput(
+interface FileSystemTransactionSourceDrift {
+  /**
+   * Differences that make the copy an unfaithful image of the opened file, or
+   * that mean the path no longer names it. Always fatal.
+   */
+  hard: string[]
+  /**
+   * Differences a re-copy can settle: an inode attribute re-stamp on an
+   * otherwise identical file, or content that has not stopped moving yet.
+   * Retryable.
+   */
+  soft: string[]
+}
+
+/**
+ * Compare the source as it was before the copy against the pinned descriptor
+ * and the path after it, and name every field that moved.
+ *
+ * One message used to stand for eight independent conditions, which made a
+ * failure from a platform the author cannot reach (a FreeBSD CI VM, say)
+ * impossible to diagnose: a truncated read, a chmod, a timestamp re-stamp and a
+ * replaced path all read identically.
+ */
+function describeFileSystemTransactionSourceDrift(
+  before: BigIntStats,
+  after: BigIntStats,
+  bytesRead: number,
+  pathStats: BigIntStats | undefined,
+): FileSystemTransactionSourceDrift {
+  const hard: string[] = []
+  const soft: string[] = []
+  const note = (
+    into: string[],
+    field: string,
+    beforeValue: bigint,
+    afterValue: bigint,
+  ) => {
+    if (beforeValue !== afterValue) {
+      into.push(`${field} ${beforeValue} -> ${afterValue}`)
+    }
+  }
+  note(hard, 'dev', before.dev, after.dev)
+  note(hard, 'ino', before.ino, after.ino)
+  note(hard, 'size', before.size, after.size)
+  if (after.size !== BigInt(bytesRead)) {
+    hard.push(`bytesRead ${bytesRead} != size ${after.size}`)
+  }
+  note(soft, 'mode', before.mode, after.mode)
+  note(soft, 'mtimeNs', before.mtimeNs, after.mtimeNs)
+  note(soft, 'ctimeNs', before.ctimeNs, after.ctimeNs)
+  if (pathStats === undefined) {
+    hard.push('path identity vanished')
+  } else if (!pathStats.isFile()) {
+    hard.push('path identity is no longer a regular file')
+  } else if (!statIdentitiesMatch(before, pathStats)) {
+    hard.push(
+      `path identity ${before.dev}/${before.ino} -> ${pathStats.dev}/${pathStats.ino}`,
+    )
+  }
+  return { hard, soft }
+}
+
+/**
+ * How many times a snapshot re-copies a source that moved underneath it before
+ * giving up. See the drift classification comment inside
+ * {@link snapshotFileSystemTransactionInput}.
+ */
+const fileSystemTransactionSnapshotAttempts = 3
+
+/**
+ * Exported for unit tests; not part of the supported `@napi-rs/cli` surface.
+ *
+ * @param onAfterCopy test seam invoked with the 1-based attempt number after
+ * each copy pass and before the post-copy re-verification, so a test can mutate
+ * the source deterministically instead of racing a timer against the copy.
+ */
+export async function snapshotFileSystemTransactionInput(
   source: string,
   destination: string,
   mode?: number,
@@ -4364,6 +4440,7 @@ async function snapshotFileSystemTransactionInput(
   recordDestinationIdentity?: (
     identity: FileSystemTransactionFileIdentity,
   ) => Promise<void>,
+  onAfterCopy?: (attempt: number) => Promise<void>,
 ): Promise<FileSystemTransactionJournalFileState> {
   // All stats in this flow are bigint so every path-vs-handle continuity check
   // below compares exact 64-bit identity, never the lossy Number dev/ino: a
@@ -4416,7 +4493,6 @@ async function snapshotFileSystemTransactionInput(
         'changed before it could be snapshotted',
       )
     }
-    const finalMode = mode ?? Number(sourceStats.mode & 0o7777n)
     if (createDestinationParent) {
       await mkdir(dirname(destination), { recursive: true })
     }
@@ -4463,50 +4539,116 @@ async function snapshotFileSystemTransactionInput(
         }
         await recordDestinationIdentity(destinationIdentity)
       }
-      const hash = createHash('sha256')
       const buffer = Buffer.allocUnsafe(64 * 1024)
+      // A snapshot's job is to record a *consistent* copy, so the source is
+      // re-verified after the copy against the stats taken before it. Drift
+      // splits in two:
+      //
+      //   hard — dev, ino, size, bytes-read, path identity. The copy is not a
+      //     faithful image of the file that was opened, or the path no longer
+      //     names it. Never tolerated.
+      //   soft — mode, mtimeNs, ctimeNs on an otherwise identical inode. This
+      //     is a metadata re-stamp: a kernel, a permission normalization or a
+      //     stray `utimes` moves these without touching a byte, and it is a
+      //     routine thing to happen to a file this process itself just wrote
+      //     into its own staging directory. Failing a release build over it is
+      //     wrong — the recorded sha256 plus dev/ino/size already guarantee the
+      //     content — so redo the snapshot instead.
+      //
+      // Metadata alone cannot decide that, though, so a retry also has to
+      // reproduce the bytes. Each attempt rebases the baseline on what it just
+      // observed, and a timestamp only moves as far as its filesystem can
+      // express: where the clock is coarse, or where several writes land in one
+      // tick, a writer that rewrites the file in place at the same length moves
+      // no field this function compares. Metadata would read as settled while
+      // the copy mixed two versions of the file, and the hash of that mixture
+      // would be recorded as the authoritative one — nothing downstream ever
+      // reads the source again to notice. So an attempt that follows drift is
+      // accepted only when it hashes to exactly what the attempt before it
+      // hashed to: two passes in a row agreeing on the content is the evidence
+      // the source is settled. A hash that keeps moving is named as
+      // `contentHash` drift and spends the attempt bound like any other.
+      //
+      // The source descriptor is deliberately *not* re-opened between attempts
+      // — it pins the inode validated on the way in, so a retry can never adopt
+      // a successor swapped into the path.
+      let baselineStats = sourceStats
+      let sourceHash = ''
+      let previousHash: string | undefined
       let position = 0
-      while (true) {
-        const { bytesRead } = await sourceHandle.read(
-          buffer,
-          0,
-          buffer.length,
+      let drift: FileSystemTransactionSourceDrift | undefined
+      for (
+        let attempt = 1;
+        attempt <= fileSystemTransactionSnapshotAttempts;
+        attempt++
+      ) {
+        if (attempt > 1) {
+          // Discard the previous attempt's bytes. The destination inode is
+          // transaction-owned and unpublished, and its identity — already
+          // recorded above — is unaffected by a truncate.
+          await destinationHandle.truncate(0)
+        }
+        const hash = createHash('sha256')
+        position = 0
+        while (true) {
+          const { bytesRead } = await sourceHandle.read(
+            buffer,
+            0,
+            buffer.length,
+            position,
+          )
+          if (bytesRead === 0) {
+            break
+          }
+          hash.update(buffer.subarray(0, bytesRead))
+          let written = 0
+          while (written < bytesRead) {
+            const result = await destinationHandle.write(
+              buffer,
+              written,
+              bytesRead - written,
+              position + written,
+            )
+            written += result.bytesWritten
+          }
+          position += bytesRead
+        }
+        sourceHash = hash.digest('hex')
+        await onAfterCopy?.(attempt)
+        const [finalSourceStats, finalPathStats] = await Promise.all([
+          sourceHandle.stat({ bigint: true }),
+          lstatIfExists(source, { bigint: true }),
+        ])
+        drift = describeFileSystemTransactionSourceDrift(
+          baselineStats,
+          finalSourceStats,
           position,
+          finalPathStats,
         )
-        if (bytesRead === 0) {
+        // Rebase on what was just observed either way: on success it is the
+        // settled metadata the journal should record, and on soft drift it is
+        // the baseline the next attempt has to hold still against.
+        baselineStats = finalSourceStats
+        if (previousHash !== undefined && previousHash !== sourceHash) {
+          drift.soft.push(`contentHash ${previousHash} -> ${sourceHash}`)
+        }
+        previousHash = sourceHash
+        if (drift.hard.length > 0 || drift.soft.length === 0) {
           break
         }
-        hash.update(buffer.subarray(0, bytesRead))
-        let written = 0
-        while (written < bytesRead) {
-          const result = await destinationHandle.write(
-            buffer,
-            written,
-            bytesRead - written,
-            position + written,
+      }
+      if (drift && (drift.hard.length > 0 || drift.soft.length > 0)) {
+        const reasons = [...drift.hard, ...drift.soft]
+        if (drift.hard.length === 0) {
+          reasons.push(
+            `still drifting after ${fileSystemTransactionSnapshotAttempts} snapshot attempts`,
           )
-          written += result.bytesWritten
         }
-        position += bytesRead
-      }
-      const [finalSourceStats, finalPathStats] = await Promise.all([
-        sourceHandle.stat({ bigint: true }),
-        lstatIfExists(source, { bigint: true }),
-      ])
-      if (
-        !statIdentitiesMatch(sourceStats, finalSourceStats) ||
-        finalSourceStats.size !== sourceStats.size ||
-        finalSourceStats.size !== BigInt(position) ||
-        finalSourceStats.mode !== sourceStats.mode ||
-        finalSourceStats.mtimeNs !== sourceStats.mtimeNs ||
-        finalSourceStats.ctimeNs !== sourceStats.ctimeNs ||
-        finalPathStats?.isFile() !== true ||
-        !statIdentitiesMatch(sourceStats, finalPathStats)
-      ) {
         throw new Error(
-          `Filesystem transaction source changed while it was snapshotted: ${source}`,
+          `Filesystem transaction source changed while it was snapshotted: ${source} (${reasons.join('; ')})`,
         )
       }
+      const finalMode = mode ?? Number(baselineStats.mode & 0o7777n)
       await applyFileSystemTransactionMode(destinationHandle, destinationMode)
       await destinationHandle.sync()
       const finalDestinationStats = await destinationHandle.stat({
@@ -4538,7 +4680,7 @@ async function snapshotFileSystemTransactionInput(
       const sourceIdentityStats = await sourceHandle.stat({ bigint: true })
       return {
         dev: String(sourceIdentityStats.dev),
-        hash: hash.digest('hex'),
+        hash: sourceHash,
         ino: String(sourceIdentityStats.ino),
         mode: finalMode,
       }