@mengruo/dsh-vision-toolkit 0.0.1 → 0.1.0

package/src/runtime-install.ts

@@ -22,9 +22,9 @@ import {
   utimes,
   writeFile,
 } from 'node:fs/promises'
-import { createWriteStream } from 'node:fs'
+import { createWriteStream, type Dirent } from 'node:fs'
 import { homedir } from 'node:os'
-import { dirname, join, resolve } from 'node:path'
+import { basename, dirname, join, resolve } from 'node:path'
 import { Transform } from 'node:stream'
 import { pipeline } from 'node:stream/promises'
 import { fileURLToPath } from 'node:url'
@@ -88,6 +88,9 @@ const PYPI_MIRROR_BASE_URL = 'https://mirrors.cloud.tencent.com/pypi/simple'
 const PROBE_TIMEOUT_MS = 30_000
 const LOCK_STALE_MS = 15 * 60 * 1000
 const LOCK_HEARTBEAT_MS = 5_000
+const WINDOWS_FILE_RETRY_ATTEMPTS = 5
+const WINDOWS_FILE_RETRY_DELAY_MS = 250
+const LEGACY_RUNTIME_GC_STALE_MS = 24 * 60 * 60 * 1000
 
 /** Absolute root of the packaged upstream snapshot. */
 export function bundledUpstreamRoot(): string {
@@ -103,6 +106,207 @@ function sha256(bytes: string | Buffer): string {
   return createHash('sha256').update(bytes).digest('hex')
 }
 
+/**
+ * Windows Defender/antivirus real-time scanning briefly locks freshly written
+ * Python DLLs, so recursive removal and directory replacement can fail with
+ * EBUSY/EPERM immediately after installation. Retry those transient Windows
+ * errors before surfacing them; non-Windows platforms pass through unchanged.
+ */
+export async function withWindowsTransientRetry<T>(operation: () => Promise<T>): Promise<T> {
+  let lastError: unknown
+  for (let attempt = 1; attempt <= WINDOWS_FILE_RETRY_ATTEMPTS; attempt += 1) {
+    try {
+      return await operation()
+    } catch (error) {
+      lastError = error
+      const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined
+      const transient = process.platform === 'win32' && (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES')
+      if (!transient) break
+      if (attempt < WINDOWS_FILE_RETRY_ATTEMPTS) {
+        await new Promise(resolveWait => setTimeout(resolveWait, WINDOWS_FILE_RETRY_DELAY_MS * attempt))
+      }
+    }
+  }
+  throw lastError
+}
+
+/**
+ * Best-effort removal used after the primary runtime path has already
+ * succeeded or failed. Transient Windows locks must not turn a usable runtime
+ * into an error, but leaving the directory behind should still be audible.
+ */
+export async function ignoreCleanupFailure(ctx: Context, label: string, path: string): Promise<void> {
+  try {
+    await withWindowsTransientRetry(() => rm(path, { recursive: true, force: true }))
+  } catch (error) {
+    ctx.logger.warn(
+      'dsh-vision-toolkit: %s cleanup failed: %s',
+      label,
+      error instanceof Error ? error.message : String(error),
+    )
+  }
+}
+
+type RuntimeGarbageKind = 'managed runtime staging' | 'managed runtime quarantine' | 'bundled Python staging'
+
+interface RuntimeGarbageCandidate {
+  kind: RuntimeGarbageKind
+  lockName?: string
+  lockToken?: string
+  minimumAgeMs?: number
+  createdAtMs?: number
+  observationMarker?: string
+}
+
+function runtimeGarbageLockToken(lockBase: string): string {
+  return sha256(lockBase).slice(0, 12)
+}
+
+function managedRuntimeGarbage(name: string): RuntimeGarbageCandidate | undefined {
+  if (name.startsWith('.prepare-')) {
+    const current = /^\.prepare-([a-f0-9]{12})-[^/]{6}$/.exec(name)
+    return { kind: 'managed runtime staging', ...(current === null ? {} : { lockToken: current[1] }) }
+  }
+  const replaced = name.indexOf('.replaced-')
+  if (replaced > 0) {
+    const stamped = /^(\d{13})-/.exec(name.slice(replaced + '.replaced-'.length))
+    return {
+      kind: 'managed runtime quarantine',
+      lockName: `${name.slice(0, replaced)}.lock`,
+      minimumAgeMs: LEGACY_RUNTIME_GC_STALE_MS,
+      ...(stamped === null
+        ? { observationMarker: '.dsh-vision-toolkit-gc-observed' }
+        : { createdAtMs: Number(stamped[1]) }),
+    }
+  }
+  return undefined
+}
+
+function bundledPythonGarbage(name: string): RuntimeGarbageCandidate | undefined {
+  if (!name.startsWith('.python-bootstrap-')) return undefined
+  const current = /^\.python-bootstrap-([a-f0-9]{12})-[^/]{6}$/.exec(name)
+  return { kind: 'bundled Python staging', ...(current === null ? {} : { lockToken: current[1] }) }
+}
+
+async function runtimeLockIsActive(lockPath: string, now: number): Promise<boolean> {
+  try {
+    const info = await stat(lockPath)
+    return now - info.mtimeMs <= LOCK_STALE_MS
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false
+    return true
+  }
+}
+
+async function garbageCollectDirectory(
+  ctx: Context,
+  parent: string,
+  classify: (name: string) => RuntimeGarbageCandidate | undefined,
+  now: number,
+): Promise<void> {
+  let entries: Dirent[]
+  try {
+    entries = await readdir(parent, { withFileTypes: true })
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return
+    ctx.logger.warn(
+      'dsh-vision-toolkit: runtime garbage collection scan failed for %s: %s',
+      parent,
+      error instanceof Error ? error.message : String(error),
+    )
+    return
+  }
+  let activeLockTokens: Set<string> | undefined
+  const getActiveLockTokens = async (): Promise<Set<string>> => {
+    if (activeLockTokens !== undefined) return activeLockTokens
+    activeLockTokens = new Set<string>()
+    for (const lock of entries) {
+      if (
+        lock.isDirectory()
+        && lock.name.endsWith('.lock')
+        && await runtimeLockIsActive(join(parent, lock.name), now)
+      ) {
+        activeLockTokens.add(runtimeGarbageLockToken(lock.name.slice(0, -'.lock'.length)))
+      }
+    }
+    return activeLockTokens
+  }
+  let legacyCollectionSafe: boolean | undefined
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+    const candidate = classify(entry.name)
+    if (candidate === undefined) continue
+    const path = join(parent, entry.name)
+    if (candidate.lockName !== undefined) {
+      if (await runtimeLockIsActive(join(parent, candidate.lockName), now)) continue
+    } else if (candidate.lockToken !== undefined) {
+      if ((await getActiveLockTokens()).has(candidate.lockToken)) continue
+    } else {
+      if (legacyCollectionSafe === undefined) {
+        legacyCollectionSafe = (await getActiveLockTokens()).size === 0
+      }
+      if (!legacyCollectionSafe) continue
+      try {
+        const info = await stat(path)
+        if (now - info.mtimeMs <= LEGACY_RUNTIME_GC_STALE_MS) continue
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue
+        ctx.logger.warn(
+          'dsh-vision-toolkit: runtime garbage collection inspection failed for %s: %s',
+          path,
+          error instanceof Error ? error.message : String(error),
+        )
+        continue
+      }
+    }
+    if (candidate.minimumAgeMs !== undefined) {
+      let createdAt = candidate.createdAtMs
+      if (createdAt === undefined && candidate.observationMarker !== undefined) {
+        const marker = join(path, candidate.observationMarker)
+        try {
+          createdAt = (await stat(marker)).mtimeMs
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+            await writeFile(marker, `${now}\n`, { flag: 'wx' }).catch(() => {})
+          }
+          continue
+        }
+      }
+      if (createdAt === undefined) {
+        try {
+          createdAt = (await stat(path)).mtimeMs
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue
+          ctx.logger.warn(
+            'dsh-vision-toolkit: runtime garbage collection inspection failed for %s: %s',
+            path,
+            error instanceof Error ? error.message : String(error),
+          )
+          continue
+        }
+      }
+      if (now - createdAt <= candidate.minimumAgeMs) continue
+    }
+    await ignoreCleanupFailure(ctx, `stale ${candidate.kind}`, path)
+  }
+}
+
+/**
+ * Opportunistically remove abandoned runtime staging and quarantine trees.
+ * Current names encode their owning runtime lock, so live preparation is
+ * skipped. Quarantines retain a 24-hour recovery window; legacy names use the
+ * same grace period and are collected only when no runtime lock is active
+ * because they cannot be associated with a specific lock.
+ */
+export async function garbageCollectRuntimeCache(
+  ctx: Context,
+  stateRoot: string,
+  now: number = Date.now(),
+): Promise<void> {
+  await garbageCollectDirectory(ctx, join(stateRoot, 'python'), managedRuntimeGarbage, now)
+  await garbageCollectDirectory(ctx, join(stateRoot, 'python-bootstrap'), bundledPythonGarbage, now)
+}
+
 export function isolatedPythonEnvironment(home: string): NodeJS.ProcessEnv {
   return {
     HOME: home,
@@ -488,7 +692,7 @@ async function withDirectoryLock<T>(lockPath: string, fn: () => Promise<T>): Pro
     await writeFile(join(lockPath, 'owner'), `${owner}\n`, { flag: 'wx' })
   } catch (error) {
     if (acquired) {
-      await rm(lockPath, { recursive: true, force: true }).catch(() => {})
+      await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true })).catch(() => {})
       throw error
     }
     if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
@@ -497,7 +701,7 @@ async function withDirectoryLock<T>(lockPath: string, fn: () => Promise<T>): Pro
       try {
         const info = await stat(lockPath)
         if (Date.now() - info.mtimeMs > LOCK_STALE_MS) {
-          await rm(lockPath, { recursive: true, force: true })
+          await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true }))
           return withDirectoryLock(lockPath, fn)
         }
       } catch {
@@ -518,7 +722,7 @@ async function withDirectoryLock<T>(lockPath: string, fn: () => Promise<T>): Pro
     clearInterval(heartbeat)
     try {
       if ((await readFile(join(lockPath, 'owner'), 'utf8')).trim() === owner) {
-        await rm(lockPath, { recursive: true, force: true })
+        await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true }))
       }
     } catch {
       // The lock was already removed or replaced.
@@ -552,8 +756,8 @@ export async function acquireBundledPython(
     if (ready !== undefined) return
     const parent = dirname(root)
     await mkdir(parent, { recursive: true })
-    await rm(root, { recursive: true, force: true })
-    const work = await mkdtemp(join(parent, '.python-bootstrap-'))
+    await withWindowsTransientRetry(() => rm(root, { recursive: true, force: true }))
+    const work = await mkdtemp(join(parent, `.python-bootstrap-${runtimeGarbageLockToken(basename(root))}-`))
     try {
       const archive = join(work, 'python.tar.gz')
       const extractDir = join(work, 'extract')
@@ -589,9 +793,9 @@ export async function acquireBundledPython(
         )
       }
       if (process.platform !== 'win32') await chmod(extractedInterpreter, 0o755)
-      await rename(extractDir, root)
+      await withWindowsTransientRetry(() => rename(extractDir, root))
     } finally {
-      await rm(work, { recursive: true, force: true })
+      await ignoreCleanupFailure(ctx, 'bundled Python staging', work)
     }
   })
   const metadata = await pythonMetadata(ctx, command, cwd)
@@ -632,6 +836,7 @@ export async function resolveBootstrapPython(
   try {
     const stateRoot = visionToolkitStateRoot()
     await mkdir(stateRoot, { recursive: true })
+    await garbageCollectRuntimeCache(ctx, stateRoot)
     const bundled = await acquireBundledPython(ctx, stateRoot, cwd, manifestOverride, requestImpl)
     return {
       command: bundled.command,
@@ -827,7 +1032,7 @@ async function waitForManagedRuntime(
     try {
       const info = await stat(lockPath)
       if (Date.now() - info.mtimeMs > LOCK_STALE_MS) {
-        await rm(lockPath, { recursive: true, force: true })
+        await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true }))
         return undefined
       }
     } catch {
@@ -844,7 +1049,7 @@ async function releaseManagedLock(lockPath: string, owner: string): Promise<void
   } catch {
     return
   }
-  await rm(lockPath, { recursive: true, force: true })
+  await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true }))
 }
 
 async function prepareManaged(
@@ -854,6 +1059,7 @@ async function prepareManaged(
 ): Promise<PreparedUpstreamRuntime> {
   const stateRoot = visionToolkitStateRoot()
   await mkdir(stateRoot, { recursive: true })
+  await garbageCollectRuntimeCache(ctx, stateRoot)
   const cleanHome = join(stateRoot, 'home')
   await mkdir(cleanHome, { recursive: true })
   const bootstrap = await resolveBootstrapPython(ctx, config.runtime.python, cleanHome)
@@ -899,7 +1105,7 @@ async function prepareManaged(
     await writeFile(join(lockPath, 'owner'), `${lockOwner}\n`, { flag: 'wx' })
   } catch (error) {
     if (lockAcquired) {
-      await rm(lockPath, { recursive: true, force: true })
+      await withWindowsTransientRetry(() => rm(lockPath, { recursive: true, force: true })).catch(() => {})
       throw error
     }
     if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
@@ -923,7 +1129,7 @@ async function prepareManaged(
     return prepareManaged(ctx, config, manifest)
   }
 
-  const staging = await mkdtemp(join(parent, '.prepare-'))
+  const staging = await mkdtemp(join(parent, `.prepare-${runtimeGarbageLockToken(runtimeId)}-`))
   const installEnv: NodeJS.ProcessEnv = {
     ...isolatedPythonEnvironment(cleanHome),
     UV_CACHE_DIR: join(stateRoot, 'uv-cache'),
@@ -1013,20 +1219,20 @@ async function prepareManaged(
       manager,
     }
     await writeFile(join(staging, 'runtime.json'), `${JSON.stringify(marker, null, 2)}\n`)
-    const quarantine = `${finalRoot}.replaced-${randomUUID()}`
+    const quarantine = `${finalRoot}.replaced-${Date.now()}-${randomUUID()}`
     let quarantined = false
     try {
-      await rename(finalRoot, quarantine)
+      await withWindowsTransientRetry(() => rename(finalRoot, quarantine))
       quarantined = true
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
     }
     try {
-      await rename(staging, finalRoot)
+      await withWindowsTransientRetry(() => rename(staging, finalRoot))
     } catch (error) {
       if (quarantined) {
         try {
-          await rename(quarantine, finalRoot)
+          await withWindowsTransientRetry(() => rename(quarantine, finalRoot))
         } catch (restoreError) {
           throw new VisionToolkitError(
             'runtime',
@@ -1037,13 +1243,18 @@ async function prepareManaged(
       }
       throw error
     }
-    await rm(quarantine, { recursive: true, force: true })
+    await ignoreCleanupFailure(ctx, 'managed runtime quarantine', quarantine)
     const python: RuntimeCommand = { program: interpreter, prefix: [], display: interpreter }
     return { source: 'managed', root: BUNDLED_ROOT, python, cleanHome, pythonVersion: metadata.version, dependencies }
   } finally {
     clearInterval(heartbeat)
-    await rm(staging, { recursive: true, force: true })
-    await releaseManagedLock(lockPath, lockOwner)
+    await ignoreCleanupFailure(ctx, 'managed runtime staging', staging)
+    await releaseManagedLock(lockPath, lockOwner).catch(error => {
+      ctx.logger.warn(
+        'dsh-vision-toolkit: managed runtime lock cleanup failed: %s',
+        error instanceof Error ? error.message : String(error),
+      )
+    })
   }
 }