@bitkyc08/opencodex 2.52.0-preview.20260912 → 2.53.0-preview.20260913

package/src/storage/cleanup/paths.ts

@@ -0,0 +1,189 @@
+import {
+  closeSync,
+  fsyncSync,
+  linkSync,
+  mkdirSync,
+  openSync,
+  unlinkSync,
+  writeFileSync,
+  chmodSync,
+} from "node:fs";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { randomUUID } from "node:crypto";
+import { renameAtomicFile } from "../../lib/windows-atomic-replace";
+
+export const ARCHIVED_SESSIONS_DIR = "archived_sessions";
+
+export const TRASH_DIR = ".trash";
+
+const JSONL_SUFFIX = ".jsonl";
+
+export const ZST_SUFFIX = ".jsonl.zst";
+
+export function chmodPrivatePath(path: string, mode: number): void {
+  try { chmodSync(path, mode); } catch { /* best-effort (e.g. Windows ACLs) */ }
+}
+
+/** Publish complete stage metadata without truncating the last recovery record. */
+export function writePrivateFile(
+  path: string,
+  content: string,
+  beforeRename?: (temporaryPath: string, targetPath: string) => void,
+): void {
+  const temporaryPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  let descriptor: number | undefined;
+  let created = false;
+  try {
+    descriptor = openSync(temporaryPath, "wx", 0o600);
+    created = true;
+    writeFileSync(descriptor, content, "utf8");
+    fsyncSync(descriptor);
+    closeSync(descriptor);
+    descriptor = undefined;
+    chmodPrivatePath(temporaryPath, 0o600);
+    beforeRename?.(temporaryPath, path);
+    renameAtomicFile(temporaryPath, path, undefined, "storage-cleanup");
+    chmodPrivatePath(path, 0o600);
+    fsyncDirectoryBestEffort(dirname(path));
+  } finally {
+    if (descriptor !== undefined) {
+      try { closeSync(descriptor); } catch { /* preserve publication failure */ }
+    }
+    if (created) {
+      try { unlinkSync(temporaryPath); } catch { /* renamed or cleanup unavailable */ }
+    }
+  }
+}
+
+/** Create `.trash/<epoch>` exclusively; suffix on collision. */
+export function createExclusiveStageDir(codexHome: string, epoch: number): string {
+  const trashRoot = join(codexHome, TRASH_DIR);
+  mkdirSync(trashRoot, { recursive: true });
+  chmodPrivatePath(trashRoot, 0o700);
+  for (let attempt = 0; attempt < 100; attempt++) {
+    const name = attempt === 0 ? String(epoch) : `${epoch}-${attempt}`;
+    const stageDir = join(trashRoot, name);
+    try {
+      mkdirSync(stageDir);
+      chmodPrivatePath(stageDir, 0o700);
+      return stageDir;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "EEXIST") continue;
+      throw error;
+    }
+  }
+  throw new Error("stage_dir_collision");
+}
+
+export function isSafeArchiveFileName(name: string): boolean {
+  if (name.includes("/") || name.includes("\\") || name.includes("..")) return false;
+  return isRolloutFileName(name);
+}
+
+export function toForwardSlash(p: string): string {
+  return p.split(sep).join("/");
+}
+
+/** Strip trailing `.zst` so plain + compressed share one logical rollout id. */
+export function logicalRolloutRelPath(relPath: string): string {
+  const normalized = toForwardSlash(relPath);
+  return normalized.endsWith(ZST_SUFFIX)
+    ? normalized.slice(0, -".zst".length)
+    : normalized;
+}
+
+export function isRolloutFileName(name: string): boolean {
+  return name.endsWith(ZST_SUFFIX) || name.endsWith(JSONL_SUFFIX);
+}
+
+/**
+ * Normalize a DB `rollout_path` to a CODEX_HOME-relative forward-slash path, then
+ * to the logical `.jsonl` form. Returns null when the path is not under
+ * `archived_sessions/` (rejects active `sessions/` and foreign paths).
+ */
+export function normalizeArchivedRolloutPath(rolloutPath: string, codexHome: string): string | null {
+  const raw = toForwardSlash(rolloutPath.trim());
+  if (!raw) return null;
+  let relativePath = raw;
+  try {
+    // Prefer Node's absolute-path detection. Do not treat a colon anywhere in the
+    // filename (Codex ISO timestamps) as an absolute Windows path.
+    const looksAbsolute = isAbsolute(raw) || /^[A-Za-z]:[\\/]/.test(raw);
+    const abs = looksAbsolute ? resolve(raw) : resolve(codexHome, raw);
+    const homeAbs = resolve(codexHome);
+    const rel = toForwardSlash(relative(homeAbs, abs));
+    if (rel.startsWith("..") || rel === "") return null;
+    relativePath = rel;
+  } catch {
+    return null;
+  }
+  const logical = logicalRolloutRelPath(relativePath);
+  if (!logical.startsWith(`${ARCHIVED_SESSIONS_DIR}/`)) return null;
+  if (!logical.endsWith(JSONL_SUFFIX)) return null;
+  // Reject path tricks: only a single file under archived_sessions/
+  const rest = logical.slice(ARCHIVED_SESSIONS_DIR.length + 1);
+  if (!rest || rest.includes("/") || rest.includes("..")) return null;
+  return logical;
+}
+
+/**
+ * Same-volume move that never replaces an existing destination.
+ *
+ * `existsSync` + `renameSync` is TOCTOU: a live file created between the check
+ * and rename can be overwritten (Windows rename replaces files). Hard-link then
+ * unlink fails with EEXIST if `to` appears, which is what trash → archived_sessions
+ * restore needs. Callers under the same `CODEX_HOME` volume should not hit EXDEV.
+ */
+export function renameNoReplace(from: string, to: string): void {
+  try {
+    linkSync(from, to);
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException | undefined)?.code;
+    // Hard links unavailable (rare FS) — refuse rather than clobber via rename.
+    if (code === "EXDEV" || code === "EPERM" || code === "ENOTSUP" || code === "EINVAL") {
+      throw Object.assign(new Error("rename_no_replace_unsupported"), { code, cause: error });
+    }
+    throw error;
+  }
+  try {
+    unlinkSync(from);
+  } catch (error) {
+    // Roll back the hard link so we do not leave the file at both paths.
+    try { unlinkSync(to); } catch { /* best-effort */ }
+    throw error;
+  }
+}
+
+export function isExistError(error: unknown): boolean {
+  return (error as NodeJS.ErrnoException | undefined)?.code === "EEXIST";
+}
+
+/**
+ * Best-effort directory fsync so a preceding rename is durable on crash.
+ * Unsupported on some Windows setups — never treat failure as fatal.
+ */
+function fsyncDirectoryBestEffort(dirPath: string): void {
+  let fd: number | undefined;
+  try {
+    fd = openSync(dirPath, "r");
+    fsyncSync(fd);
+  } catch {
+    /* best-effort */
+  } finally {
+    if (fd !== undefined) {
+      try { closeSync(fd); } catch { /* */ }
+    }
+  }
+}
+
+/** Epoch dir names: digits, optionally `-N` from createExclusiveStageDir collision. */
+export const TRASH_EPOCH_DIR = /^(\d+)(-\d+)?$/;
+
+export function isSafeArchivedPhysicalRel(rel: string): boolean {
+  const normalized = toForwardSlash(rel);
+  if (!normalized.startsWith(`${ARCHIVED_SESSIONS_DIR}/`)) return false;
+  if (normalized.includes("..")) return false;
+  const rest = normalized.slice(ARCHIVED_SESSIONS_DIR.length + 1);
+  if (!rest || rest.includes("/")) return false;
+  return isRolloutFileName(rest);
+}

package/src/storage/cleanup/pending.ts

@@ -0,0 +1,140 @@
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { join } from "node:path";
+import { renameAtomicFile } from "../../lib/windows-atomic-replace";
+import { TRASH_DIR, TRASH_EPOCH_DIR, chmodPrivatePath } from "./paths";
+
+/** Accepted destination paths from every valid in-progress restore marker under `.trash`. */
+export function collectRestorePendingAcceptedDestRels(codexHome: string): Set<string> {
+  const out = new Set<string>();
+  const trashRoot = join(codexHome, TRASH_DIR);
+  if (!existsSync(trashRoot)) return out;
+  for (const name of readdirSync(trashRoot)) {
+    if (!TRASH_EPOCH_DIR.test(name)) continue;
+    const read = readRestorePending(join(trashRoot, name));
+    if (read.status !== "valid") continue;
+    for (const rel of read.state.acceptedDestRels) out.add(rel);
+  }
+  return out;
+}
+
+/** Marks an incomplete restore so retries can accept dest files and resume metadata. */
+export const RESTORE_PENDING_FILE = "restore-pending.json";
+
+export interface RestorePendingSections {
+  state: boolean;
+  logs: boolean;
+  memories: boolean;
+  goals: boolean;
+}
+
+export interface RestorePendingState {
+  version: 1;
+  filesRestored: true;
+  /**
+   * Planned CODEX_HOME-relative destinations for this restore attempt.
+   * Written before moves so a mid-loop failure can still accept placed dests
+   * on resume while finishing files that remain staged.
+   */
+  acceptedDestRels: string[];
+  /** Sections that still need reconciliation on retry. */
+  pending: RestorePendingSections;
+}
+
+type RestorePendingRead =
+  | { status: "missing" }
+  | { status: "valid"; state: RestorePendingState }
+  | { status: "invalid" };
+
+let _restorePendingSeq = 0;
+
+function parseRestorePendingState(raw: unknown): RestorePendingState | null {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const o = raw as Record<string, unknown>;
+  if (o.version !== 1 || o.filesRestored !== true) return null;
+  if (!Array.isArray(o.acceptedDestRels)) return null;
+  const acceptedDestRels = o.acceptedDestRels.filter((r): r is string => typeof r === "string");
+  if (acceptedDestRels.length !== o.acceptedDestRels.length) return null;
+  const pendingRaw = o.pending;
+  if (!pendingRaw || typeof pendingRaw !== "object" || Array.isArray(pendingRaw)) return null;
+  const p = pendingRaw as Record<string, unknown>;
+  if (
+    typeof p.state !== "boolean"
+    || typeof p.logs !== "boolean"
+    || typeof p.memories !== "boolean"
+    || typeof p.goals !== "boolean"
+  ) {
+    return null;
+  }
+  return {
+    version: 1,
+    filesRestored: true,
+    acceptedDestRels,
+    pending: {
+      state: p.state,
+      logs: p.logs,
+      memories: p.memories,
+      goals: p.goals,
+    },
+  };
+}
+
+/**
+ * Distinguish a missing marker from a present-but-malformed one. An invalid marker
+ * must never be treated as a fresh restore (that would ignore already-moved files).
+ */
+export function readRestorePending(stageDir: string): RestorePendingRead {
+  const path = join(stageDir, RESTORE_PENDING_FILE);
+  if (!existsSync(path)) return { status: "missing" };
+  try {
+    const state = parseRestorePendingState(JSON.parse(readFileSync(path, "utf8")) as unknown);
+    if (!state) return { status: "invalid" };
+    return { status: "valid", state };
+  } catch {
+    return { status: "invalid" };
+  }
+}
+
+/**
+ * Atomically replace restore-pending.json: private temp in the stage, fsync, then rename.
+ * An interrupted update leaves the previous valid marker intact.
+ */
+export function writeRestorePending(
+  stageDir: string,
+  state: RestorePendingState,
+  options?: { failBeforeRename?: boolean; failWrite?: boolean },
+): void {
+  if (options?.failWrite) throw new Error("test_fail_pending_write");
+  const dest = join(stageDir, RESTORE_PENDING_FILE);
+  const tmp = join(stageDir, `${RESTORE_PENDING_FILE}.${process.pid}.${++_restorePendingSeq}.tmp`);
+  const payload = JSON.stringify(state);
+  const fd = openSync(tmp, "w", 0o600);
+  try {
+    writeSync(fd, payload, null, "utf8");
+    fsyncSync(fd);
+  } catch (error) {
+    try { closeSync(fd); } catch { /* */ }
+    try { unlinkSync(tmp); } catch { /* */ }
+    throw error;
+  }
+  closeSync(fd);
+  chmodPrivatePath(tmp, 0o600);
+  if (options?.failBeforeRename) {
+    try { unlinkSync(tmp); } catch { /* */ }
+    throw new Error("test_fail_pending_rename");
+  }
+  try {
+    renameAtomicFile(tmp, dest, undefined, "storage-cleanup");
+  } catch (error) {
+    try { unlinkSync(tmp); } catch { /* */ }
+    throw error;
+  }
+}

package/src/storage/cleanup/preview.ts

@@ -0,0 +1,292 @@
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+import { resolveCodexHomeDir } from "../../codex/home";
+import { columnExists, discoverRuntimeDbPaths, tableExists } from "./db";
+import { ARCHIVED_SESSIONS_DIR, isSafeArchiveFileName, logicalRolloutRelPath, normalizeArchivedRolloutPath } from "./paths";
+import { collectRestorePendingAcceptedDestRels } from "./pending";
+import type { ArchivedCandidate, CleanupPreview } from "./types";
+
+export function clampPercent(percent: unknown): number {
+  if (typeof percent !== "number" || !Number.isFinite(percent)) return 0;
+  return Math.max(0, Math.min(100, Math.floor(percent)));
+}
+
+function candidateDigestLines(candidates: ArchivedCandidate[]): string[] {
+  return candidates
+    .map(c => {
+      const physical = [...c.physicalFiles]
+        .sort((a, b) => a.relPath.localeCompare(b.relPath))
+        .map(f => `${f.relPath}|${f.bytes}|${Math.trunc(f.mtimeMs)}`)
+        .join(",");
+      return `${c.relPath}|${c.bytes}|${Math.trunc(c.mtimeMs)}|${physical}`;
+    })
+    .sort();
+}
+
+/** Content digest of the exact previewed candidate set (paths + size + mtime). */
+export function computePreviewDigest(candidates: ArchivedCandidate[], percent: number): string {
+  return createHash("sha256")
+    .update(`${clampPercent(percent)}\n${candidateDigestLines(candidates).join("\n")}`)
+    .digest("hex");
+}
+
+/**
+ * Digest bound to an explicit candidate list (not a percent selection).
+ * Used when reduceToBytes needs an exact count that percent rounding cannot represent.
+ */
+export function computeExactPreviewDigest(candidates: ArchivedCandidate[]): string {
+  return createHash("sha256")
+    .update(`exact\n${candidateDigestLines(candidates).join("\n")}`)
+    .digest("hex");
+}
+
+/** List archived rollout groups oldest-first. Never walks `sessions/`. */
+export function listArchivedCandidates(codexHome: string): ArchivedCandidate[] {
+  const dir = join(codexHome, ARCHIVED_SESSIONS_DIR);
+  let names: string[] = [];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return [];
+  }
+
+  type Acc = {
+    logicalRel: string;
+    files: Array<{ name: string; absPath: string; relPath: string; bytes: number; mtimeMs: number }>;
+  };
+  const groups = new Map<string, Acc>();
+
+  for (const name of names) {
+    if (!isSafeArchiveFileName(name)) continue;
+    const absPath = join(dir, name);
+    try {
+      const st = statSync(absPath);
+      if (!st.isFile()) continue;
+      const relPath = `${ARCHIVED_SESSIONS_DIR}/${name}`;
+      const logicalRel = logicalRolloutRelPath(relPath);
+      let acc = groups.get(logicalRel);
+      if (!acc) {
+        acc = { logicalRel, files: [] };
+        groups.set(logicalRel, acc);
+      }
+      acc.files.push({
+        name,
+        absPath,
+        relPath,
+        bytes: st.size,
+        mtimeMs: st.mtimeMs,
+      });
+    } catch {
+      /* vanished mid-scan */
+    }
+  }
+
+  const out: ArchivedCandidate[] = [];
+  for (const acc of groups.values()) {
+    // Prefer the plain `.jsonl` path as the public/logical identity when both exist.
+    acc.files.sort((a, b) => a.relPath.localeCompare(b.relPath));
+    const primary =
+      acc.files.find(f => f.relPath === acc.logicalRel) ??
+      acc.files[0]!;
+    out.push({
+      relPath: acc.logicalRel,
+      absPath: primary.absPath,
+      bytes: acc.files.reduce((sum, f) => sum + f.bytes, 0),
+      mtimeMs: Math.min(...acc.files.map(f => f.mtimeMs)),
+      physicalRelPaths: acc.files.map(f => f.relPath),
+      physicalFiles: acc.files.map(f => ({ relPath: f.relPath, bytes: f.bytes, mtimeMs: f.mtimeMs })),
+    });
+  }
+  out.sort((a, b) => a.mtimeMs - b.mtimeMs || a.relPath.localeCompare(b.relPath));
+  return out;
+}
+
+export function selectOldestPercent(candidates: ArchivedCandidate[], percent: number): ArchivedCandidate[] {
+  const pct = clampPercent(percent);
+  if (pct <= 0 || candidates.length === 0) return [];
+  if (pct >= 100) return [...candidates];
+  const n = percentSelectionTargetCount(candidates.length, pct);
+  return candidates.slice(0, n);
+}
+
+/** Count implied by percent selection over the full candidate list. */
+export function percentSelectionTargetCount(totalCount: number, percent: number): number {
+  const pct = clampPercent(percent);
+  if (pct <= 0 || totalCount === 0) return 0;
+  if (pct >= 100) return totalCount;
+  return Math.max(1, Math.floor((totalCount * pct) / 100));
+}
+
+export function candidateOverlapsPendingRestore(
+  candidate: ArchivedCandidate,
+  pendingDestRels: ReadonlySet<string>,
+): boolean {
+  if (pendingDestRels.size === 0) return false;
+  for (const rel of candidate.physicalRelPaths) {
+    if (pendingDestRels.has(rel)) return true;
+  }
+  return pendingDestRels.has(candidate.relPath);
+}
+
+/** Drop cleanup candidates whose physical paths overlap an in-progress restore. */
+export function filterCandidatesExcludingPendingRestore(
+  candidates: ArchivedCandidate[],
+  codexHome: string = resolveCodexHomeDir(),
+): ArchivedCandidate[] {
+  const pendingDestRels = collectRestorePendingAcceptedDestRels(codexHome);
+  if (pendingDestRels.size === 0) return candidates;
+  return candidates.filter(c => !candidateOverlapsPendingRestore(c, pendingDestRels));
+}
+
+/**
+ * Oldest-first percent selection that skips pending-restore destinations without
+ * consuming the percent budget, backfilling with the next oldest safe candidates.
+ */
+export function selectOldestPercentSkippingPendingRestore(
+  candidates: ArchivedCandidate[],
+  percent: number,
+  codexHome: string = resolveCodexHomeDir(),
+): ArchivedCandidate[] {
+  const target = percentSelectionTargetCount(candidates.length, percent);
+  if (target === 0) return [];
+  const pendingDestRels = collectRestorePendingAcceptedDestRels(codexHome);
+  const out: ArchivedCandidate[] = [];
+  for (const c of candidates) {
+    if (candidateOverlapsPendingRestore(c, pendingDestRels)) continue;
+    out.push(c);
+    if (out.length >= target) break;
+  }
+  return out;
+}
+
+/**
+ * Reduce archived total toward `reduceToBytes` using oldest safe candidates only.
+ * Pending-restore destinations are skipped and do not count toward bytes freed.
+ */
+export function selectReduceToBytesSkippingPendingRestore(
+  candidates: ArchivedCandidate[],
+  reduceToBytes: number,
+  codexHome: string = resolveCodexHomeDir(),
+): ArchivedCandidate[] {
+  if (!Number.isFinite(reduceToBytes) || reduceToBytes < 0) return [];
+  const total = candidates.reduce((sum, c) => sum + c.bytes, 0);
+  if (total <= reduceToBytes) return [];
+  const need = total - reduceToBytes;
+  const pendingDestRels = collectRestorePendingAcceptedDestRels(codexHome);
+  const out: ArchivedCandidate[] = [];
+  let freed = 0;
+  for (const c of candidates) {
+    if (candidateOverlapsPendingRestore(c, pendingDestRels)) continue;
+    out.push(c);
+    freed += c.bytes;
+    if (freed >= need) break;
+  }
+  return out;
+}
+
+/**
+ * Normalized rollout paths of pinned threads. Pinned threads are never
+ * cleanup candidates: a pin is the user's explicit "keep this" signal and
+ * deleting its rollout would be permanent task-data loss (#858).
+ *
+ * Selection-time use is advisory: on any DB problem this returns an empty
+ * set, and the write-locked re-check inside reconcileDeletedThreads stays
+ * the fail-closed gate. Older schemas without `is_pinned` keep prior
+ * behavior.
+ */
+function collectPinnedArchivedRolloutPaths(codexHome: string): Set<string> {
+  const statePath = discoverRuntimeDbPaths(codexHome).state;
+  if (!statePath || !existsSync(statePath)) return new Set();
+  let db: Database | undefined;
+  try {
+    db = new Database(statePath, { readonly: true });
+    if (!tableExists(db, "threads") || !columnExists(db, "threads", "is_pinned")) {
+      return new Set();
+    }
+    const rows = db.query<{ rollout_path: string }, []>(
+      `SELECT rollout_path FROM threads WHERE is_pinned = 1`,
+    ).all();
+    const out = new Set<string>();
+    for (const row of rows) {
+      const normalized = normalizeArchivedRolloutPath(row.rollout_path, codexHome);
+      if (normalized) out.add(normalized);
+    }
+    return out;
+  } catch {
+    return new Set();
+  } finally {
+    try { db?.close(); } catch { /* */ }
+  }
+}
+
+/** Drop candidates whose rollout belongs to a pinned thread (#858). */
+export function filterCandidatesExcludingPinned(
+  candidates: ArchivedCandidate[],
+  codexHome: string,
+): ArchivedCandidate[] {
+  const pinned = collectPinnedArchivedRolloutPaths(codexHome);
+  if (pinned.size === 0) return candidates;
+  return candidates.filter(c => !pinned.has(c.relPath));
+}
+
+export function previewArchivedCleanup(
+  percent: number,
+  codexHome: string = resolveCodexHomeDir(),
+): CleanupPreview {
+  const all = listArchivedCandidates(codexHome);
+  const safe = selectOldestPercentSkippingPendingRestore(
+    filterCandidatesExcludingPinned(all, codexHome),
+    percent,
+    codexHome,
+  );
+  const pct = clampPercent(percent);
+  return {
+    codexHome,
+    percent: pct,
+    count: safe.length,
+    bytes: safe.reduce((sum, c) => sum + c.bytes, 0),
+    digest: computePreviewDigest(safe, pct),
+    candidates: safe,
+  };
+}
+
+/** Preview bound to an explicit candidate set (exact digest, percent left at 0). */
+export function previewExactArchivedCleanup(
+  candidates: ArchivedCandidate[],
+  codexHome: string = resolveCodexHomeDir(),
+): CleanupPreview {
+  const safe = filterCandidatesExcludingPinned(
+    filterCandidatesExcludingPendingRestore(candidates, codexHome),
+    codexHome,
+  );
+  return {
+    codexHome,
+    percent: 0,
+    count: safe.length,
+    bytes: safe.reduce((sum, c) => sum + c.bytes, 0),
+    digest: computeExactPreviewDigest(safe),
+    candidates: safe,
+  };
+}
+
+/**
+ * Resolve an exact candidate list from current archive state.
+ * Returns null when any requested path is missing or drifted (caller maps to stale_preview).
+ */
+export function resolveExactArchivedCandidates(
+  candidateRelPaths: string[],
+  codexHome: string = resolveCodexHomeDir(),
+): ArchivedCandidate[] | null {
+  if (!Array.isArray(candidateRelPaths) || candidateRelPaths.length === 0) return [];
+  const all = listArchivedCandidates(codexHome);
+  const byRel = new Map(all.map(c => [c.relPath, c]));
+  const selected: ArchivedCandidate[] = [];
+  for (const rel of candidateRelPaths) {
+    const hit = byRel.get(rel);
+    if (!hit) return null;
+    selected.push(hit);
+  }
+  return selected;
+}