@tikoci/rosetta 0.8.9 → 0.8.11

package/src/setup.ts

@@ -8,7 +8,18 @@
  */
 
 import { execSync } from "node:child_process";
-import { existsSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import {
+  closeSync,
+  existsSync,
+  openSync,
+  readdirSync,
+  readSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import path from "node:path";
 import { gunzipSync } from "bun";
 import { detectMode, resolveBaseDir, resolveDbPath, resolveVersion, SCHEMA_VERSION } from "./paths.ts";
 
@@ -25,20 +36,51 @@ const MIN_PAGES = 100;
 const MIN_COMMANDS = 1000;
 const MIN_DECOMPRESSED_BYTES = 50 * 1024 * 1024; // 50 MB
 const SQLITE_MAGIC = "SQLite format 3\0";
+const DOWNLOAD_LOCK_WAIT_MS = 90_000;
+const DOWNLOAD_LOCK_POLL_MS = 250;
+const DOWNLOAD_LOCK_STALE_MS = 15 * 60 * 1000;
+
+type DbProbe = {
+  schemaVersion: number;
+  pages: number;
+  commands: number;
+  releaseTag: string | null;
+};
+
+type DownloadLockHandle = {
+  fd: number;
+  path: string;
+};
 
 /** Check if a DB file exists and has actual page data.
  *  Opens read-write — see probeDb's note: freshly written WAL-mode files can
  *  fail to open readonly on macOS when the .shm file is missing. */
 function dbHasData(dbPath: string): boolean {
+  return hasMinimumDbContent(probeDb(dbPath));
+}
+
+function looksLikeSqliteFile(dbPath: string): boolean {
   if (!existsSync(dbPath)) return false;
+
+  let fd: number | null = null;
   try {
-    const { default: sqlite } = require("bun:sqlite");
-    const check = new sqlite(dbPath);
-    const row = check.prepare("SELECT COUNT(*) AS c FROM pages").get() as { c: number };
-    check.close();
-    return row.c > 0;
+    const stats = statSync(dbPath);
+    if (!stats.isFile() || stats.size < SQLITE_MAGIC.length) return false;
+
+    fd = openSync(dbPath, "r");
+    const header = Buffer.alloc(SQLITE_MAGIC.length);
+    const bytesRead = readSync(fd, header, 0, header.byteLength, 0);
+    return bytesRead === header.byteLength && header.toString("utf8") === SQLITE_MAGIC;
   } catch {
     return false;
+  } finally {
+    if (fd !== null) {
+      try {
+        closeSync(fd);
+      } catch {
+        // best-effort cleanup
+      }
+    }
   }
 }
 
@@ -54,6 +96,8 @@ export function probeDb(dbPath: string): {
   commands: number;
   releaseTag: string | null;
 } | null {
+  if (!looksLikeSqliteFile(dbPath)) return null;
+
   try {
     const { default: sqlite } = require("bun:sqlite");
     const check = new sqlite(dbPath);
@@ -79,6 +123,155 @@ export function probeDb(dbPath: string): {
   }
 }
 
+export function hasMinimumDbContent(probe: DbProbe | null): probe is DbProbe {
+  return !!probe && probe.pages >= MIN_PAGES && probe.commands >= MIN_COMMANDS;
+}
+
+export function isUsableDbProbe(probe: DbProbe | null): probe is DbProbe {
+  return hasMinimumDbContent(probe) && probe.schemaVersion === SCHEMA_VERSION;
+}
+
+function lockPathFor(dbPath: string): string {
+  return `${dbPath}.lock`;
+}
+
+function formatProbeSummary(probe: DbProbe): string {
+  const tagInfo = probe.releaseTag ? ` (release ${probe.releaseTag})` : "";
+  return `schema v${probe.schemaVersion}, ${probe.pages} pages, ${probe.commands} commands${tagInfo}`;
+}
+
+export function tryAcquireDownloadLock(dbPath: string): DownloadLockHandle | null {
+  const lockPath = lockPathFor(dbPath);
+
+  while (true) {
+    try {
+      const fd = openSync(lockPath, "wx");
+      writeFileSync(
+        fd,
+        `${JSON.stringify({
+          pid: process.pid,
+          created_at: new Date().toISOString(),
+          db_path: dbPath,
+        })}\n`,
+      );
+      return { fd, path: lockPath };
+    } catch (e) {
+      const code = e instanceof Error && "code" in e ? e.code : undefined;
+      if (code !== "EEXIST") throw e;
+
+      let ageMs: number | null = null;
+      try {
+        ageMs = Date.now() - statSync(lockPath).mtimeMs;
+      } catch {
+        ageMs = null;
+      }
+
+      if (ageMs !== null && ageMs > DOWNLOAD_LOCK_STALE_MS) {
+        tryUnlink(lockPath);
+        continue;
+      }
+
+      return null;
+    }
+  }
+}
+
+export function releaseDownloadLock(lock: DownloadLockHandle | null): void {
+  if (!lock) return;
+  try {
+    closeSync(lock.fd);
+  } catch {
+    // best-effort cleanup
+  }
+  tryUnlink(lock.path);
+}
+
+export async function waitForUsableDb(
+  dbPath: string,
+  log: (msg: string) => void = console.log,
+  timeoutMs = DOWNLOAD_LOCK_WAIT_MS,
+): Promise<boolean> {
+  const lockPath = lockPathFor(dbPath);
+  const deadline = Date.now() + timeoutMs;
+  let announced = false;
+
+  while (Date.now() < deadline) {
+    if (!existsSync(lockPath)) {
+      return isUsableDbProbe(probeDb(dbPath));
+    }
+
+    if (!announced) {
+      log(`  Another rosetta process is preparing ${dbPath}; waiting...`);
+      announced = true;
+    }
+
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) break;
+    await Bun.sleep(Math.min(remaining, DOWNLOAD_LOCK_POLL_MS));
+  }
+
+  return false;
+}
+
+function tryUnlinkDbSidecars(dbPath: string): void {
+  tryUnlink(`${dbPath}-wal`);
+  tryUnlink(`${dbPath}-shm`);
+}
+
+function cleanupDbArtifacts(dbPath: string): void {
+  tryUnlinkDbSidecars(dbPath);
+  tryUnlink(dbPath);
+}
+
+export function cleanupStaleTempArtifacts(dbPath: string, staleMs = DOWNLOAD_LOCK_STALE_MS): number {
+  const dir = path.dirname(dbPath);
+  const prefix = `${path.basename(dbPath)}.tmp.`;
+  const now = Date.now();
+  let removed = 0;
+
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return 0;
+  }
+
+  for (const entry of entries) {
+    if (!entry.startsWith(prefix)) continue;
+
+    const fullPath = path.join(dir, entry);
+    try {
+      const ageMs = now - statSync(fullPath).mtimeMs;
+      if (ageMs <= staleMs) continue;
+    } catch {
+      continue;
+    }
+
+    try {
+      unlinkSync(fullPath);
+      removed++;
+    } catch {
+      // best-effort cleanup
+    }
+  }
+
+  return removed;
+}
+
+function replaceDbFile(tmpPath: string, dbPath: string): void {
+  try {
+    renameSync(tmpPath, dbPath);
+    return;
+  } catch (e) {
+    const code = e instanceof Error && "code" in e ? e.code : undefined;
+    // Windows can return EBUSY, EPERM, or EEXIST when the destination is open.
+    if (code !== "EBUSY" && code !== "EEXIST" && code !== "EPERM") throw e;
+  }
+
+  tryUnlink(dbPath);
+  renameSync(tmpPath, dbPath);
+}
+
 /** Build the version-pinned download URL. Falls back to /latest/ when no version.
  *  Exported for test coverage. */
 export function dbDownloadUrls(version: string): string[] {
@@ -96,7 +289,7 @@ export function dbDownloadUrls(version: string): string[] {
  * Download ros-help.db.gz from GitHub Releases atomically:
  *   1. Try version-pinned URL first, fall back to /latest/ on 404.
  *   2. Decompress in memory, verify SQLite magic bytes + minimum size.
- *   3. Write to <dbPath>.tmp.<pid>, open it read-only, verify schema_version
+ *   3. Write to <dbPath>.tmp.<pid>, probe it with SQLite, verify schema_version
  *      matches the running code and pages/commands counts look healthy.
  *   4. Atomically rename .tmp → dbPath, then delete stale .db-wal / .db-shm.
  *
@@ -107,122 +300,171 @@ export function dbDownloadUrls(version: string): string[] {
 export async function downloadDb(
   dbPath: string,
   log: (msg: string) => void = console.log,
-) {
-  const urls = dbDownloadUrls(RELEASE_VERSION);
-  let lastError: Error | null = null;
-
-  for (let i = 0; i < urls.length; i++) {
-    const url = urls[i];
-    const isLast = i === urls.length - 1;
-    log(`Downloading database from GitHub Releases...`);
-    log(`  ${url}`);
-
-    let response: Response;
-    try {
-      response = await fetch(url, { redirect: "follow" });
-    } catch (e) {
-      lastError = e as Error;
-      log(`  Network error: ${e}`);
-      if (isLast) throw lastError;
-      continue;
+): Promise<DbProbe> {
+  let lock = tryAcquireDownloadLock(dbPath);
+  if (!lock) {
+    const reused = await waitForUsableDb(dbPath, log);
+    if (reused) {
+      const probe = probeDb(dbPath);
+      if (probe) {
+        log(`  Reused existing database: ${formatProbeSummary(probe)}`);
+        return probe;
+      }
     }
 
-    if (response.status === 404 && !isLast) {
-      log(`  Not found at this URL, trying fallback...`);
-      continue;
+    // Re-probe once — the lock may have been released with a healthy DB
+    const fallbackProbe = probeDb(dbPath);
+    if (isUsableDbProbe(fallbackProbe)) {
+      log(`  Reused existing database: ${formatProbeSummary(fallbackProbe)}`);
+      return fallbackProbe;
     }
-    if (!response.ok) {
-      lastError = new Error(`Download failed: ${response.status} ${response.statusText}`);
-      if (isLast) throw lastError;
-      log(`  ${lastError.message} — trying fallback...`);
-      continue;
-    }
-
-    const contentLength = response.headers.get("content-length");
-    const totalMB = contentLength ? (Number(contentLength) / 1024 / 1024).toFixed(1) : "?";
-    log(`  Downloading ${totalMB} MB (compressed)...`);
-
-    const compressed = new Uint8Array(await response.arrayBuffer());
-    log(`  Decompressing...`);
 
-    let decompressed: Uint8Array;
-    try {
-      decompressed = gunzipSync(compressed);
-    } catch (e) {
-      lastError = new Error(`Gunzip failed (corrupt download or HTML error page): ${e}`);
-      if (isLast) throw lastError;
-      log(`  ${lastError.message}`);
-      continue;
-    }
-
-    // Validate magic bytes and minimum size before touching the filesystem.
-    if (decompressed.byteLength < MIN_DECOMPRESSED_BYTES) {
-      lastError = new Error(
-        `Decompressed DB too small: ${decompressed.byteLength} bytes (expected ≥ ${MIN_DECOMPRESSED_BYTES})`,
+    lock = tryAcquireDownloadLock(dbPath);
+    if (!lock) {
+      throw new Error(
+        `Timed out waiting for another rosetta process to finish preparing ${dbPath}. ` +
+          `Close other rosetta clients and retry.`,
       );
-      if (isLast) throw lastError;
-      log(`  ${lastError.message}`);
-      continue;
-    }
-    const header = new TextDecoder().decode(decompressed.subarray(0, SQLITE_MAGIC.length));
-    if (header !== SQLITE_MAGIC) {
-      lastError = new Error("Downloaded payload is not a SQLite database (magic bytes mismatch)");
-      if (isLast) throw lastError;
-      log(`  ${lastError.message}`);
-      continue;
     }
+  }
 
-    // Write to a temp file next to the canonical DB path, validate, then rename.
-    const tmpPath = `${dbPath}.tmp.${process.pid}`;
-    try {
-      writeFileSync(tmpPath, decompressed);
-    } catch (e) {
-      lastError = new Error(`Write to ${tmpPath} failed: ${e}`);
-      throw lastError;
-    }
+  const urls = dbDownloadUrls(RELEASE_VERSION);
+  let lastError: Error | null = null;
 
-    const probe = probeDb(tmpPath);
-    if (!probe) {
-      tryUnlink(tmpPath);
-      lastError = new Error("Downloaded DB failed to open with SQLite");
-      if (isLast) throw lastError;
-      log(`  ${lastError.message} — trying fallback...`);
-      continue;
-    }
-    if (probe.schemaVersion !== SCHEMA_VERSION) {
-      tryUnlink(tmpPath);
-      lastError = new Error(
-        `Downloaded DB schema=${probe.schemaVersion} does not match this rosetta build (expected ${SCHEMA_VERSION}). ` +
-          `This usually means the cached package version is older than the published DB. ` +
-          `Run \`bun pm cache rm\` and relaunch to pick up the latest package.`,
-      );
-      if (isLast) throw lastError;
-      log(`  ${lastError.message}`);
-      continue;
-    }
-    if (probe.pages < MIN_PAGES || probe.commands < MIN_COMMANDS) {
-      tryUnlink(tmpPath);
-      lastError = new Error(
-        `Downloaded DB content looks incomplete (pages=${probe.pages}, commands=${probe.commands})`,
-      );
-      if (isLast) throw lastError;
-      log(`  ${lastError.message} — trying fallback...`);
-      continue;
+  try {
+    const cleaned = cleanupStaleTempArtifacts(dbPath);
+    if (cleaned > 0) {
+      log(`  Removed ${cleaned} stale temp DB artifact${cleaned === 1 ? "" : "s"}.`);
     }
 
-    // Validation passed — drop stale WAL/SHM and atomically swap.
-    tryUnlink(`${dbPath}-wal`);
-    tryUnlink(`${dbPath}-shm`);
-    renameSync(tmpPath, dbPath);
+    for (let i = 0; i < urls.length; i++) {
+      const url = urls[i];
+      const isLast = i === urls.length - 1;
+      log(`Downloading database from GitHub Releases...`);
+      log(`  ${url}`);
+
+      let response: Response;
+      try {
+        response = await fetch(url, { redirect: "follow" });
+      } catch (e) {
+        lastError = e as Error;
+        log(`  Network error: ${e}`);
+        if (isLast) throw lastError;
+        continue;
+      }
+
+      if (response.status === 404 && !isLast) {
+        log(`  Not found at this URL, trying fallback...`);
+        continue;
+      }
+      if (!response.ok) {
+        lastError = new Error(`Download failed: ${response.status} ${response.statusText}`);
+        if (isLast) throw lastError;
+        log(`  ${lastError.message} — trying fallback...`);
+        continue;
+      }
+
+      const contentLength = response.headers.get("content-length");
+      const totalMB = contentLength ? (Number(contentLength) / 1024 / 1024).toFixed(1) : "?";
+      log(`  Downloading ${totalMB} MB (compressed)...`);
+
+      const compressed = new Uint8Array(await response.arrayBuffer());
+      log(`  Decompressing...`);
+
+      let decompressed: Uint8Array;
+      try {
+        decompressed = gunzipSync(compressed);
+      } catch (e) {
+        lastError = new Error(`Gunzip failed (corrupt download or HTML error page): ${e}`);
+        if (isLast) throw lastError;
+        log(`  ${lastError.message}`);
+        continue;
+      }
+
+      // Validate magic bytes and minimum size before touching the filesystem.
+      if (decompressed.byteLength < MIN_DECOMPRESSED_BYTES) {
+        lastError = new Error(
+          `Decompressed DB too small: ${decompressed.byteLength} bytes (expected ≥ ${MIN_DECOMPRESSED_BYTES})`,
+        );
+        if (isLast) throw lastError;
+        log(`  ${lastError.message}`);
+        continue;
+      }
+      const header = new TextDecoder().decode(decompressed.subarray(0, SQLITE_MAGIC.length));
+      if (header !== SQLITE_MAGIC) {
+        lastError = new Error("Downloaded payload is not a SQLite database (magic bytes mismatch)");
+        if (isLast) throw lastError;
+        log(`  ${lastError.message}`);
+        continue;
+      }
+
+      // Write to a temp file next to the canonical DB path, validate, then rename.
+      const tmpPath = `${dbPath}.tmp.${process.pid}`;
+      try {
+        writeFileSync(tmpPath, decompressed);
+      } catch (e) {
+        lastError = new Error(`Write to ${tmpPath} failed: ${e}`);
+        throw lastError;
+      }
+
+      const probe = probeDb(tmpPath);
+      if (!probe) {
+        cleanupDbArtifacts(tmpPath);
+        lastError = new Error("Downloaded DB failed to open with SQLite");
+        if (isLast) throw lastError;
+        log(`  ${lastError.message} — trying fallback...`);
+        continue;
+      }
+      if (probe.schemaVersion !== SCHEMA_VERSION) {
+        cleanupDbArtifacts(tmpPath);
+        lastError = new Error(
+          `Downloaded DB schema=${probe.schemaVersion} does not match this rosetta build (expected ${SCHEMA_VERSION}). ` +
+            `This usually means the cached package version is older than the published DB. ` +
+            `Run \`bun pm cache rm\` and relaunch to pick up the latest package.`,
+        );
+        if (isLast) throw lastError;
+        log(`  ${lastError.message}`);
+        continue;
+      }
+      if (probe.pages < MIN_PAGES || probe.commands < MIN_COMMANDS) {
+        cleanupDbArtifacts(tmpPath);
+        lastError = new Error(
+          `Downloaded DB content looks incomplete (pages=${probe.pages}, commands=${probe.commands})`,
+        );
+        if (isLast) throw lastError;
+        log(`  ${lastError.message} — trying fallback...`);
+        continue;
+      }
+
+      // Validation passed — drop stale WAL/SHM and atomically swap.
+      try {
+        tryUnlinkDbSidecars(tmpPath);
+        tryUnlinkDbSidecars(dbPath);
+        replaceDbFile(tmpPath, dbPath);
+      } catch (e) {
+        const existingProbe = probeDb(dbPath);
+        if (hasMinimumDbContent(existingProbe) && existingProbe.schemaVersion === probe.schemaVersion && existingProbe.releaseTag === probe.releaseTag) {
+          cleanupDbArtifacts(tmpPath);
+          log(`  Another rosetta process already installed the same database.`);
+          log(`  Reused existing database: ${formatProbeSummary(existingProbe)}`);
+          return existingProbe;
+        }
+
+        cleanupDbArtifacts(tmpPath);
+        throw e;
+      }
+
+      const sizeMB = (decompressed.byteLength / 1024 / 1024).toFixed(1);
+      const tagInfo = probe.releaseTag ? ` (release ${probe.releaseTag})` : "";
+      log(`  Wrote ${sizeMB} MB to ${dbPath}${tagInfo}`);
+      log(`  Validated: schema v${probe.schemaVersion}, ${probe.pages} pages, ${probe.commands} commands.`);
+      return probe;
+    }
 
-    const sizeMB = (decompressed.byteLength / 1024 / 1024).toFixed(1);
-    const tagInfo = probe.releaseTag ? ` (release ${probe.releaseTag})` : "";
-    log(`  Wrote ${sizeMB} MB to ${dbPath}${tagInfo}`);
-    log(`  Validated: schema v${probe.schemaVersion}, ${probe.pages} pages, ${probe.commands} commands.`);
-    return;
+    throw lastError ?? new Error("Database download failed for unknown reasons");
+  } finally {
+    releaseDownloadLock(lock);
   }
-
-  throw lastError ?? new Error("Database download failed for unknown reasons");
 }
 
 /** Remove a file if it exists, swallowing all errors. */
@@ -241,17 +483,13 @@ function tryUnlink(p: string): void {
  */
 export async function refreshDb(log: (msg: string) => void = console.log): Promise<boolean> {
   const dbPath = resolveDbPath(import.meta.dirname);
+  let probe: DbProbe;
   try {
-    await downloadDb(dbPath, log);
+    probe = await downloadDb(dbPath, log);
   } catch (e) {
     log(`✗ Refresh failed: ${e instanceof Error ? e.message : e}`);
     return false;
   }
-  const probe = probeDb(dbPath);
-  if (!probe) {
-    log(`✗ Post-download probe failed`);
-    return false;
-  }
   const tagInfo = probe.releaseTag ? ` (release ${probe.releaseTag})` : "";
   log(`✓ Database ready${tagInfo}: ${probe.pages} pages, ${probe.commands} commands, schema v${probe.schemaVersion}`);
   return true;
@@ -260,6 +498,7 @@ export async function refreshDb(log: (msg: string) => void = console.log): Promi
 export async function runSetup(force = false) {
   const mode = detectMode(import.meta.dirname);
   const dbPath = resolveDbPath(import.meta.dirname);
+  let downloadedProbe: DbProbe | null = null;
 
   console.log(`rosetta ${RELEASE_VERSION}`);
   console.log(`  ${link("https://github.com/tikoci/rosetta")}`);
@@ -272,7 +511,7 @@ export async function runSetup(force = false) {
     console.log(`  (use --refresh or --setup --force to re-download)`);
   } else {
     try {
-      await downloadDb(dbPath);
+      downloadedProbe = await downloadDb(dbPath);
     } catch (e) {
       console.error(`✗ Database download failed: ${e instanceof Error ? e.message : e}`);
       process.exit(1);
@@ -281,7 +520,7 @@ export async function runSetup(force = false) {
 
   // ── Validate DB ──
   console.log();
-  const probe = probeDb(dbPath);
+  const probe = downloadedProbe ?? probeDb(dbPath);
   if (!probe) {
     console.error(`✗ Database validation failed: cannot open ${dbPath}`);
     const retryCmd = mode === "compiled" ? "rosetta" : mode === "package" ? "bunx @tikoci/rosetta" : "bun run src/setup.ts";
@@ -293,7 +532,7 @@ export async function runSetup(force = false) {
       `✗ DB schema version is ${probe.schemaVersion}, expected ${SCHEMA_VERSION}.`,
     );
     console.error(
-      `  Cached package may be out of date. Run \`bun pm cache rm\` and relaunch.`,
+      `  Package may be out of date. Run: bunx @tikoci/rosetta@latest --refresh`,
     );
     process.exit(1);
   }