@bitkyc08/opencodex 2.77.0 → 2.78.0

package/src/codex/catalog/retained-sync.ts

@@ -2,7 +2,7 @@ import { projectAntigravitySelectedModels } from "../../providers/antigravity-ef
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { initializeConfigOwnership } from "../../lib/config-ownership";
-import { getConfigDir, loadConfig, websocketsEnabled } from "../../config";
+import { getConfigDir, loadConfig, websocketsEnabled, withConfigMutationLockSync } from "../../config";
 import { shouldSyncCodexOnStart } from "../desired-state";
 import { legacyCustomModelCatalogSlugs } from "../custom-model-catalog-migration";
 import { getCodexHome } from "../paths";
@@ -40,6 +40,8 @@ import {
   nativeMultiAgentDefaults,
 } from "./parsing";
 import type { CatalogModel, MultiAgentMode, RawCatalog, RawEntry } from "./parsing";
+import { ocxRoutedNamespaceCounts, routedRemovalBackedByConfigFile, unconfiguredRoutedRemoval } from "./routed-removal";
+import { ConfigMutationLockError } from "../../config/mutation-lock";
 import {
   accountBoundNativeOpenAiSlugsBySelector,
   desktopAllowlistSuppressedNativeSlugs,
@@ -64,11 +66,13 @@ import {
 } from "../catalog-write-serialization";
 import {
   preparedBytesDifferFromDisk,
+  auditRefusedCatalogReplacement,
   publishHashedCodexCatalogBackup,
   publishLegacyCodexCatalogBackup,
   replaceActiveCodexCatalog,
   replaceCodexModelsCache,
   type PreparedCatalogFileWrite,
+  type CatalogFileReplacement,
 } from "../internal/catalog-writer";
 import { visibleCodexAccountSelectors } from "./account-models";
 import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS, NATIVE_OPENAI_MODELS, NATIVE_RESERVE_MODEL } from "./native-models";
@@ -105,8 +109,15 @@ interface RetainedCatalogSyncResult {
   comboOmissions: ComboCatalogOmission[];
   /** Validated catalog commit (including identical bytes), or a refused refresh. */
   refreshOutcome?: "committed" | "refused";
-  /** `desired_disabled` observed under K after the provider await; nothing was written. */
-  skippedReason?: "desired_disabled";
+  /**
+   * `desired_disabled`: observed under K after the provider await. `unbacked_routed_removal`: the
+   * catalog would lose routed namespaces config.json still enables, or config.json is missing or
+   * unreadable (#6529). `foreign_owner`: this Codex home is bound to another OPENCODEX_HOME. Nothing
+   * was written in any of these cases.
+   */
+  skippedReason?: "desired_disabled" | "unbacked_routed_removal" | "foreign_owner" | "owner_unknown";
+  /** With `unbacked_routed_removal`: how many routed namespaces the refused write would have emptied. */
+  protectedRoutedNamespaces?: number;
 }
 
 /**
@@ -554,12 +565,46 @@ function writeRetainedCatalogSync({
   if (!preparedBytesDifferFromDisk(preparedCatalog)) {
     return { added, path: catalogPath, catalogWritten: false, comboOmissions };
   }
-
-  replaceActiveCodexCatalog(permit, owningCodexHome, preparedCatalog);
+  // A refresh may drop a provider's rows only when config.json on disk agrees the provider is gone.
+  // Without that, a config that is not the user's (missing or unreadable file read as defaults,
+  // another OPENCODEX_HOME) would publish a native-only catalog and exit cleanly (#6529).
+  const refused = (protectedRoutedNamespaces: number): RetainedCatalogSyncResult => ({
+    added: 0,
+    path: catalogPath,
+    catalogWritten: false,
+    comboOmissions,
+    skippedReason: "unbacked_routed_removal",
+    protectedRoutedNamespaces,
+  });
+  let replacement: CatalogFileReplacement | undefined;
+  const removal = unconfiguredRoutedRemoval(onDiskCatalog, catalog, config);
+  if (removal !== null) {
+    // Hold C (K -> C) from the config read through the replacement, so a save that enables one of
+    // these providers cannot land between the check and the write. A held C is a refusal too.
+    let backed = false;
+    try {
+      backed = withConfigMutationLockSync(() => {
+        if (!routedRemovalBackedByConfigFile(removal)) return false;
+        replacement = replaceActiveCodexCatalog(permit, owningCodexHome, preparedCatalog);
+        return replacement.kind !== "refused";
+      });
+    } catch (error) {
+      if (!(error instanceof ConfigMutationLockError)) throw error;
+    }
+    if (!backed) {
+      if (replacement?.kind !== "refused") {
+        auditRefusedCatalogReplacement(permit, owningCodexHome, preparedCatalog, "unbacked-routed-removal");
+      }
+      return refused(removal.namespaces.length);
+    }
+  } else {
+    replacement = replaceActiveCodexCatalog(permit, owningCodexHome, preparedCatalog);
+    if (replacement.kind === "refused") return refused(ocxRoutedNamespaceCounts(onDiskCatalog).size);
+  }
   return {
     added,
     path: catalogPath,
-    catalogWritten: true,
+    catalogWritten: replacement?.kind === "written",
     comboOmissions,
   };
 }
@@ -649,7 +694,7 @@ export async function syncCatalogModels(
       owningCodexHome,
       modelEntitlements,
     });
-  });
+  }, { intent: "refresh", writer: "retained-sync" });
   if (committed.kind === "completed" && committed.value !== null) {
     return {
       ...committed.value,
@@ -662,6 +707,10 @@ export async function syncCatalogModels(
     catalogWritten: false,
     comboOmissions,
     refreshOutcome: "refused",
+    ...(committed.kind === "unavailable" && committed.reason === "foreign-owner"
+      ? { skippedReason: "foreign_owner" as const } : {}),
+    ...(committed.kind === "unavailable" && committed.reason === "owner-unknown"
+      ? { skippedReason: "owner_unknown" as const } : {}),
   };
 }
 
@@ -729,9 +778,7 @@ export function invalidateCodexModelsCacheWithPermitOutcome(
     // `cacheSynced` mean what its name and its consumers already assume, and what
     // `pullRemoteCatalog` and the early returns in `refreshCodexModelCatalog`
     // already assert: a write happened.
-    if (!preparedBytesDifferFromDisk(preparedCache)) return "unchanged";
-    replaceCodexModelsCache(permit, owningCodexHome, preparedCache);
-    return "written";
+    return replaceCodexModelsCache(permit, owningCodexHome, preparedCache).kind;
   } catch {
     return "failed";
   }
@@ -750,6 +797,7 @@ export function invalidateCodexModelsCache(options?: CodexCatalogSyncOptions): b
   const outcome = withCatalogWriteSerialization(
     owningCodexHome,
     permit => invalidateCodexModelsCacheWithPermit(permit, owningCodexHome, options),
+    { intent: "cache", writer: "cache-invalidate" },
   );
   return outcome.kind === "completed" && outcome.value;
 }

package/src/codex/catalog/routed-removal.ts

@@ -0,0 +1,116 @@
+import { COMBO_NAMESPACE } from "../../combos";
+import { readConfigAdmissionSnapshot, type ConfigAdmissionSnapshot } from "../../config/diagnostics";
+import type { OcxConfig } from "../../types";
+import { trustedAccountBoundNativeCatalogSlug } from "./account-models";
+import { isOcxAuthoredRoutedEntry } from "./build-entries";
+import { isNativeAliasCatalogEntry } from "./metadata";
+import type { RawCatalog, RawEntry } from "./parsing";
+
+/**
+ * Routed namespaces a candidate catalog empties of OpenCodex-authored rows only because the config
+ * that produced it does not enable them: the "deleted provider" rule in build-entries. That rule is
+ * the one way a refresh removes a provider's rows wholesale, so it is the one a config that is not
+ * the user's (a missing or unreadable config.json read as defaults, another OPENCODEX_HOME) turns
+ * into a silent native-only catalog (#6529).
+ */
+export interface UnconfiguredRoutedRemoval {
+  readonly namespaces: readonly string[];
+}
+
+type RoutedNamespaceConfig = Pick<OcxConfig, "providers" | "combos">;
+
+function routedNamespace(entry: RawEntry): string | null {
+  if (!entry || typeof entry !== "object" || typeof entry.slug !== "string") return null;
+  // Account-bound native rows follow the account pool, not provider config.
+  if (trustedAccountBoundNativeCatalogSlug(entry) !== undefined) return null;
+  if (isNativeAliasCatalogEntry(entry)) return COMBO_NAMESPACE;
+  // Combo aliases may be bare or use an unrelated provider namespace. Ownership selects the
+  // combo namespace only with the same generated-description authorship signal as routed rows.
+  if (entry.owned_by === COMBO_NAMESPACE
+    && typeof entry.description === "string"
+    && entry.description.startsWith("Routed via opencodex → ")) return COMBO_NAMESPACE;
+  // Foreign rows (Cursor, user tooling) are never removed by the provider rule.
+  if (!isOcxAuthoredRoutedEntry(entry)) return null;
+  return entry.slug.slice(0, entry.slug.indexOf("/"));
+}
+
+/** OpenCodex-authored routed rows per namespace; foreign and account-bound rows never count. */
+export function ocxRoutedNamespaceCounts(catalog: RawCatalog | null): Map<string, number> {
+  const counts = new Map<string, number>();
+  for (const entry of catalog?.models ?? []) {
+    const namespace = routedNamespace(entry);
+    if (namespace !== null) counts.set(namespace, (counts.get(namespace) ?? 0) + 1);
+  }
+  return counts;
+}
+
+/** Total OpenCodex-authored routed rows, or null for a value that is not a catalog. */
+export function ocxRoutedRowCount(catalog: unknown): number | null {
+  if (catalog === null || typeof catalog !== "object" || !Array.isArray((catalog as RawCatalog).models)) return null;
+  let total = 0;
+  for (const count of ocxRoutedNamespaceCounts(catalog as RawCatalog).values()) total += count;
+  return total;
+}
+
+/** Whether a config still produces routed rows for a namespace: an enabled provider, or any combo. */
+export function configEnablesRoutedNamespace(config: RoutedNamespaceConfig, namespace: string): boolean {
+  if (namespace === COMBO_NAMESPACE && Object.keys(config.combos ?? {}).length > 0) return true;
+  const provider = config.providers?.[namespace];
+  return provider !== undefined && provider.disabled !== true;
+}
+
+/**
+ * The namespaces whose OpenCodex-authored routed rows are on disk now, absent from the candidate,
+ * and not enabled by the driving config; null when the candidate removes no such namespace.
+ * Removals inside a namespace the driving config still enables (discovery, selection, a deleted
+ * custom model) are the provider's own business and are not reported here.
+ */
+export function unconfiguredRoutedRemoval(
+  active: RawCatalog | null,
+  candidate: RawCatalog,
+  driving: RoutedNamespaceConfig,
+): UnconfiguredRoutedRemoval | null {
+  const activeModels = active?.models ?? [];
+  if (activeModels.length === 0) return null;
+  const kept = new Set((candidate.models ?? []).flatMap(entry => typeof entry.slug === "string" ? [entry.slug] : []));
+  const namespaces = new Set<string>();
+  for (const entry of activeModels) {
+    if (typeof entry.slug === "string" && kept.has(entry.slug)) continue;
+    const namespace = routedNamespace(entry);
+    if (namespace !== null && !configEnablesRoutedNamespace(driving, namespace)) namespaces.add(namespace);
+  }
+  return namespaces.size > 0 ? { namespaces: [...namespaces].sort() } : null;
+}
+
+/**
+ * True when config.json on disk backs every removal: it is a readable, parseable file, and it enables
+ * none of the removed namespaces. A missing, unreadable or salvaged file backs nothing, and a file
+ * that still enables a namespace shows the driving config was not the one on disk. Read once, at
+ * write time, so a config that fell back to defaults while the file was briefly unreadable cannot
+ * pass on a later successful read.
+ */
+export function routedRemovalBackedByConfigFile(
+  removal: UnconfiguredRoutedRemoval,
+  read: () => ConfigAdmissionSnapshot = readConfigAdmissionSnapshot,
+): boolean {
+  const snapshot = read();
+  if (snapshot.kind !== "read" || snapshot.diagnostics.source !== "file") return false;
+  return removal.namespaces.every(namespace => !configEnablesRoutedNamespace(snapshot.diagnostics.config, namespace));
+}
+
+/** One line for logs when K refused a writer from another OPENCODEX_HOME; names no path. */
+export const FOREIGN_CODEX_HOME_OWNER_MESSAGE = "Codex catalog left unchanged: this Codex home was set up by "
+  + "another OpenCodex home (OPENCODEX_HOME), recorded in its opencodex-journal.json. Run opencodex "
+  + "from that home, or run `ocx restore` there first.";
+
+/** Unavailable evidence is not proof of a foreign owner. No paths or journal content are exposed. */
+export const UNKNOWN_CODEX_HOME_OWNER_MESSAGE = "Codex catalog left unchanged: ownership of this Codex home "
+  + "could not be checked. Restore readable ownership evidence and retry.";
+
+/** One line for logs: counts only, never provider names, model ids or paths. */
+export function unbackedRoutedRemovalMessage(count: number): string {
+  return `Codex catalog left unchanged: this refresh would remove the routed models of ${count} `
+    + `provider namespace${count === 1 ? "" : "s"} that config.json still enables, or config.json `
+    + "is missing or unreadable. Run `ocx sync` again once config.json is readable; use `ocx restore` "
+    + "to remove routed models on purpose.";
+}

package/src/codex/catalog/write-audit-contract.ts

@@ -0,0 +1,21 @@
+/** Bottom-layer metadata contract: neither K nor the filesystem adapter owns intent. */
+export type CatalogWriteIntent = "refresh" | "cache" | "pull" | "restore";
+export type CatalogAuditConfigSource = "file" | "default" | "fallback" | "unreadable";
+export type CatalogAuditRefusalReason = "foreign-owner" | "owner-unknown"
+  | "unbacked-routed-removal" | "unbacked-routed-clear";
+
+export interface CatalogWriteAuditDetails {
+  readonly target: "catalog" | "cache";
+  readonly outcome: "written" | "refused";
+  readonly reason?: CatalogAuditRefusalReason;
+  readonly routedBefore?: number | null;
+  readonly routedAfter?: number | null;
+  readonly configSource?: CatalogAuditConfigSource;
+}
+
+export interface CatalogWriteAuditEvent extends CatalogWriteAuditDetails {
+  readonly intent: CatalogWriteIntent;
+  readonly writer: string;
+  /** Supplied by K, never discovered by the adapter. */
+  readonly opencodexHome: string;
+}

package/src/codex/catalog/write-audit.ts

@@ -0,0 +1,171 @@
+import {
+  closeSync, constants, fstatSync, ftruncateSync, lstatSync, openSync, readSync, writeSync,
+  type Stats,
+} from "node:fs";
+import { join } from "node:path";
+import { redactUserPath } from "../../lib/redact";
+import { hardenSecretPath, windowsSecretAclApplies } from "../../lib/windows-secret-acl";
+import type { CatalogWriteAuditEvent } from "./write-audit-contract";
+
+export const CODEX_CATALOG_AUDIT_FILE = "opencodex-catalog-audit.jsonl";
+export const CATALOG_AUDIT_MAX_BYTES = 256 * 1024;
+export const CATALOG_AUDIT_KEEP_RECORDS = 400;
+export const CATALOG_AUDIT_MAX_EVENT_BYTES = 2 * 1024;
+const MAX_HOME_CHARACTERS = 256;
+const AUDIT_ACL_DEADLINE_MS = 1000;
+const WRITERS = new Set([
+  "convergence", "retained-sync", "cache-from-catalog", "cache-invalidate",
+  "catalog-pull", "catalog-restore", "codex-restore", "sync-cache", "startup-cache",
+]);
+
+export function codexCatalogAuditPath(codexHome: string): string {
+  return join(codexHome, CODEX_CATALOG_AUDIT_FILE);
+}
+
+function safeCount(value: number | null | undefined): number | null {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+
+function auditLine(event: CatalogWriteAuditEvent): Buffer {
+  const writer = WRITERS.has(event.writer) ? event.writer : "other";
+  // Explicit projection: no argv, arbitrary event properties, model IDs or provider names.
+  return Buffer.from(`${JSON.stringify({
+    at: new Date().toISOString(), pid: process.pid, ppid: process.ppid,
+    command: writer,
+    opencodexHome: redactUserPath(event.opencodexHome).slice(0, MAX_HOME_CHARACTERS),
+    target: event.target, outcome: event.outcome, reason: event.reason,
+    intent: event.intent, writer,
+    configSource: event.configSource,
+    routedBefore: safeCount(event.routedBefore), routedAfter: safeCount(event.routedAfter),
+  })}\n`);
+}
+
+function regularPrivateFile(stat: Stats): boolean {
+  // POSIX bits cannot prove NTFS privacy. Windows requires fresh-file ACL hardening.
+  return !windowsSecretAclApplies() && stat.isFile() && stat.nlink === 1
+    && stat.uid === process.getuid?.() && (stat.mode & 0o777) === 0o600;
+}
+
+function sameFile(left: Stats, right: Stats): boolean {
+  return left.dev === right.dev && left.ino === right.ino && left.size === right.size
+    && left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs;
+}
+
+/** Read at most the tail budget plus one byte to locate a complete first record. */
+function readTail(fd: number, size: number): Buffer {
+  const start = Math.max(0, size - CATALOG_AUDIT_MAX_BYTES - 1);
+  const bytes = Buffer.alloc(size - start);
+  let length = 0;
+  while (length < bytes.length) {
+    const count = readSync(fd, bytes, length, bytes.length - length, start + length);
+    if (count === 0) break;
+    length += count;
+  }
+  if (length !== bytes.length) throw new Error("audit file changed during read");
+  if (start === 0) return bytes;
+  const boundary = bytes.indexOf(10);
+  return boundary === -1 ? Buffer.alloc(0) : bytes.subarray(boundary + 1);
+}
+
+function completeRecords(bytes: Buffer): { records: Buffer[]; count: number; clean: boolean } {
+  const records: Buffer[] = [];
+  let start = 0;
+  let clean = true;
+  let count = 0;
+  for (let end = bytes.indexOf(10); end !== -1; end = bytes.indexOf(10, start)) {
+    const line = bytes.subarray(start, end);
+    try {
+      const text = line.toString("utf8");
+      if (!line.equals(Buffer.from(text))) throw new Error("invalid UTF-8");
+      JSON.parse(text);
+      records[count % CATALOG_AUDIT_KEEP_RECORDS] = bytes.subarray(start, end + 1);
+      count++;
+    } catch { clean = false; }
+    start = end + 1;
+  }
+  const pivot = count > CATALOG_AUDIT_KEEP_RECORDS ? count % CATALOG_AUDIT_KEEP_RECORDS : 0;
+  return { records: records.slice(pivot).concat(records.slice(0, pivot)), count,
+    clean: clean && start === bytes.length };
+}
+
+function writeAll(fd: number, bytes: Buffer, position: number | null): void {
+  let offset = 0;
+  while (offset < bytes.length) {
+    const count = writeSync(fd, bytes, offset, bytes.length - offset,
+      position === null ? null : position + offset);
+    if (count === 0) throw new Error("audit write made no progress");
+    offset += count;
+  }
+}
+
+/**
+ * Best-effort adapter. Caller MUST hold K, even for a refusal that has no permit.
+ * POSIX owners compact the descriptor; foreign callers only append within both caps.
+ * Windows only publishes into fresh owner files after required ACL hardening.
+ * This diagnostic has no replay or authorization role. Process death may leave a partial line.
+ */
+export function appendCatalogWriteAudit(
+  codexHome: string,
+  event: CatalogWriteAuditEvent,
+  options: { readonly create: boolean },
+): "appended" | "created" | "skipped" {
+  let fd: number | undefined;
+  try {
+    const line = auditLine(event);
+    if (line.length > CATALOG_AUDIT_MAX_EVENT_BYTES) return "skipped";
+    const path = codexCatalogAuditPath(codexHome);
+    const windows = windowsSecretAclApplies();
+    let before: Stats | undefined;
+    try { before = lstatSync(path); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !options.create) return "skipped";
+    }
+    if (before && !regularPrivateFile(before)) return "skipped";
+    // No mkdir, pathname truncation or chmod. O_EXCL prevents creation over a raced path.
+    const flags = constants.O_RDWR | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0)
+      | (before ? (options.create ? 0 : constants.O_APPEND) : constants.O_CREAT | constants.O_EXCL);
+    fd = openSync(path, flags, 0o600);
+    let opened = fstatSync(fd);
+    if (windows) {
+      // Existing files have already been refused: no read-only NTFS privacy verifier exists.
+      if (!opened.isFile() || opened.nlink !== 1 || opened.size !== 0
+        || !sameFile(opened, lstatSync(path))) return "skipped";
+      if (!hardenSecretPath(path, { required: true, deadlineMs: AUDIT_ACL_DEADLINE_MS }).ok) return "skipped";
+      const hardened = fstatSync(fd);
+      // ACL changes legitimately move ctime. Retain object identity, then take a fresh baseline.
+      if (!hardened.isFile() || hardened.nlink !== 1 || hardened.size !== 0
+        || opened.dev !== hardened.dev || opened.ino !== hardened.ino
+        || !sameFile(hardened, lstatSync(path))) return "skipped";
+      opened = hardened;
+    } else if (!regularPrivateFile(opened) || (before && !sameFile(before, opened))) return "skipped";
+    if (!options.create && opened.size + line.length > CATALOG_AUDIT_MAX_BYTES) return "skipped";
+    const tail = readTail(fd, opened.size);
+    const parsed = completeRecords(tail);
+    if (!sameFile(opened, fstatSync(fd)) || !sameFile(opened, lstatSync(path))) return "skipped";
+    if (!options.create) {
+      if (!parsed.clean || parsed.count + 1 > CATALOG_AUDIT_KEEP_RECORDS) return "skipped";
+      writeAll(fd, line, null);
+    } else if (opened.size <= CATALOG_AUDIT_MAX_BYTES - line.length
+      && parsed.clean && parsed.count < CATALOG_AUDIT_KEEP_RECORDS) {
+      writeAll(fd, line, opened.size);
+    } else {
+      const retained: Buffer[] = [line];
+      let bytes = line.length;
+      for (let index = parsed.records.length - 1; index >= 0 && retained.length < CATALOG_AUDIT_KEEP_RECORDS; index--) {
+        const record = parsed.records[index]!;
+        if (bytes + record.length > CATALOG_AUDIT_MAX_BYTES) break;
+        retained.unshift(record);
+        bytes += record.length;
+      }
+      const output = Buffer.concat(retained, bytes);
+      writeAll(fd, output, 0);
+      ftruncateSync(fd, output.length);
+    }
+    return before ? "appended" : "created";
+  } catch {
+    // Deliberately best-effort: an audit IO failure never changes the catalog result.
+    return "skipped";
+  } finally {
+    if (fd !== undefined) { try { closeSync(fd); } catch { /* diagnostic only */ } }
+  }
+}