@dboio/cli 0.11.3 → 0.13.2

package/src/commands/clone.js

@@ -1,14 +1,15 @@
 import { Command } from 'commander';
-import { readFile, writeFile, appendFile, mkdir, access, readdir, rename } from 'fs/promises';
+import { readFile, writeFile, appendFile, mkdir, access, readdir, rename, stat } from 'fs/promises';
 import { join, basename, extname, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import { DboClient } from '../lib/client.js';
 import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference, saveCloneSource, loadCloneSource, saveDescriptorFilenamePreference, loadDescriptorFilenamePreference, saveDescriptorContentExtractions, loadDescriptorContentExtractions, saveExtensionDocumentationMDPlacement, loadExtensionDocumentationMDPlacement } from '../lib/config.js';
 import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, SCAFFOLD_DIRS, ENTITY_DIR_NAMES, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir, resolveEntityDirPath } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
-import { buildUidFilename, detectLegacyDotUid } from '../lib/filenames.js';
+import { buildUidFilename, buildContentFileName, stripUidFromFilename, hasUidInFilename, detectLegacyDotUid } from '../lib/filenames.js';
 import { setFileTimestamps, parseServerDate } from '../lib/timestamps.js';
-import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge, isDiffable, loadBaselineForComparison, resetBaselineCache } from '../lib/diff.js';
+import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge, isDiffable, loadBaselineForComparison, resetBaselineCache, findMetadataFiles } from '../lib/diff.js';
+import { loadIgnore } from '../lib/ignore.js';
 import { checkDomainChange } from '../lib/domain-guard.js';
 import { applyTrashIcon, ensureTrashIcon } from '../lib/folder-icon.js';
 import { loadMetadataTemplates, saveMetadataTemplates, getTemplateCols, setTemplateCols, buildTemplateFromCloneRecord } from '../lib/metadata-templates.js';
@@ -43,6 +44,143 @@ function isWillDeleteFile(filename) {
   return basename(filename).startsWith(WILL_DELETE_PREFIX);
 }
 
+/**
+ * Collect all UIDs from the freshly fetched app JSON.
+ * Traverses all entity types in appJson.children.
+ *
+ * @param {object} appJson - Parsed app JSON with children map
+ * @returns {Set<string>} Set of all UIDs present on the server
+ */
+export function collectServerUids(appJson) {
+  const uids = new Set();
+  if (!appJson?.children || typeof appJson.children !== 'object') return uids;
+
+  for (const [entityName, entries] of Object.entries(appJson.children)) {
+    if (!Array.isArray(entries)) continue;
+    for (const entry of entries) {
+      if (entry && typeof entry === 'object' && entry.UID) {
+        uids.add(String(entry.UID));
+      }
+      // Output hierarchy: collect nested child UIDs too
+      if (entry && typeof entry === 'object' && entry.children) {
+        for (const childArr of Object.values(entry.children)) {
+          if (!Array.isArray(childArr)) continue;
+          for (const child of childArr) {
+            if (child && typeof child === 'object' && child.UID) {
+              uids.add(String(child.UID));
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return uids;
+}
+
+/**
+ * Detect local metadata files whose UIDs are absent from the server response,
+ * then move them and their companion files to trash/.
+ *
+ * @param {object} appJson    - Fresh app JSON from server
+ * @param {import('ignore').Ignore} ig - Ignore instance for findMetadataFiles
+ * @param {object} sync       - Parsed synchronize.json { delete, edit, add }
+ * @param {object} options    - Clone options ({ entityFilter?, verbose? })
+ */
+export async function detectAndTrashOrphans(appJson, ig, sync, options) {
+  if (options.entityFilter) return;
+  if (!appJson?.children) return;
+
+  const serverUids = collectServerUids(appJson);
+  if (serverUids.size === 0) return;
+
+  // UIDs already queued for deletion in synchronize.json
+  const stagedDeleteUids = new Set(
+    (sync.delete || []).map(e => e.UID).filter(Boolean).map(String)
+  );
+
+  const metaFiles = await findMetadataFiles(process.cwd(), ig);
+  if (metaFiles.length === 0) return;
+
+  const trashDir = join(process.cwd(), 'trash');
+  const orphans = [];
+
+  for (const metaPath of metaFiles) {
+    let meta;
+    try {
+      meta = JSON.parse(await readFile(metaPath, 'utf8'));
+    } catch {
+      continue;
+    }
+
+    if (!meta.UID) continue;
+
+    const uid = String(meta.UID);
+    if (stagedDeleteUids.has(uid)) continue;
+    if (serverUids.has(uid)) continue;
+
+    // Orphan — collect files to move
+    const metaDir = dirname(metaPath);
+    const filesToMove = [metaPath];
+
+    for (const col of (meta._contentColumns || [])) {
+      const ref = meta[col];
+      if (ref && String(ref).startsWith('@')) {
+        const refName = String(ref).substring(1);
+        const companionPath = refName.startsWith('/')
+          ? join(process.cwd(), refName)
+          : join(metaDir, refName);
+        if (await fileExists(companionPath)) {
+          filesToMove.push(companionPath);
+        }
+      }
+    }
+
+    if (meta._mediaFile && String(meta._mediaFile).startsWith('@')) {
+      const refName = String(meta._mediaFile).substring(1);
+      const mediaPath = refName.startsWith('/')
+        ? join(process.cwd(), refName)
+        : join(metaDir, refName);
+      if (await fileExists(mediaPath)) {
+        filesToMove.push(mediaPath);
+      }
+    }
+
+    orphans.push({ metaPath, uid, entity: meta._entity || 'unknown', filesToMove });
+  }
+
+  if (orphans.length === 0) return;
+
+  await mkdir(trashDir, { recursive: true });
+
+  let trashed = 0;
+
+  for (const { metaPath, uid, entity, filesToMove } of orphans) {
+    log.dim(`  Trashed: ${basename(metaPath)} (${entity}:${uid})`);
+
+    for (const filePath of filesToMove) {
+      const destBase = basename(filePath);
+      let destPath = join(trashDir, destBase);
+
+      // Collision: append timestamp suffix (same pattern as moveWillDeleteToTrash in push.js)
+      try { await stat(destPath); destPath = `${destPath}.${Date.now()}`; } catch {}
+
+      try {
+        await rename(filePath, destPath);
+        trashed++;
+      } catch (err) {
+        log.warn(`  Could not trash: ${filePath} — ${err.message}`);
+      }
+    }
+  }
+
+  if (trashed > 0) {
+    await ensureTrashIcon(trashDir);
+    log.plain('');
+    log.warn(`Moved ${orphans.length} orphaned record(s) to trash (deleted on server)`);
+  }
+}
+
 /**
  * Resolve a content Path to a directory under Bins/.
  *
@@ -144,9 +282,11 @@ export function resolveRecordPaths(entityName, record, structure, placementPref)
   }
 
   const uid = String(record.UID || record._id || 'untitled');
-  const base = buildUidFilename(name, uid);
-  const filename = ext ? `${base}.${ext}` : base;
-  const metaPath = join(dir, `${base}.metadata.json`);
+  // Companion: natural name, no UID
+  const filename = sanitizeFilename(buildContentFileName(record, uid));
+  // Metadata: still uses ~UID
+  const metaBase = buildUidFilename(name, uid);
+  const metaPath = join(dir, `${metaBase}.metadata.json`);
 
   return { dir, filename, metaPath };
 }
@@ -156,9 +296,15 @@ export function resolveRecordPaths(entityName, record, structure, placementPref)
  * Replicates logic from processMediaEntries() for collision detection.
  */
 export function resolveMediaPaths(record, structure) {
-  const filename = record.Filename || `${record.Name || record.UID}.${(record.Extension || 'bin').toLowerCase()}`;
-  const name = sanitizeFilename(filename.replace(/\.[^.]+$/, ''));
-  const ext = (record.Extension || 'bin').toLowerCase();
+  // Companion: use Filename column directly (natural name, no UID)
+  const companionFilename = sanitizeFilename(
+    record.Filename
+    || `${record.Name || record.UID}.${(record.Extension || 'bin').toLowerCase()}`
+  );
+
+  // Base name and ext for metadata naming
+  const name = sanitizeFilename(companionFilename.replace(/\.[^.]+$/, ''));
+  const ext = (record.Extension || extname(companionFilename).substring(1) || 'bin').toLowerCase();
 
   // Always place media by BinID; fall back to bins/ root
   let dir = BINS_DIR;
@@ -169,12 +315,12 @@ export function resolveMediaPaths(record, structure) {
   dir = dir.replace(/^\/+|\/+$/g, '');
   if (!dir) dir = BINS_DIR;
 
+  // Metadata: name~uid.ext.metadata.json (unchanged format)
   const uid = String(record.UID || record._id || 'untitled');
-  const base = buildUidFilename(name, uid);
-  const finalFilename = `${base}.${ext}`;
-  const metaPath = join(dir, `${finalFilename}.metadata.json`);
+  const metaBase = buildUidFilename(name, uid);
+  const metaPath = join(dir, `${metaBase}.${ext}.metadata.json`);
 
-  return { dir, filename: finalFilename, metaPath };
+  return { dir, filename: companionFilename, metaPath };
 }
 
 /**
@@ -197,6 +343,224 @@ export function resolveEntityDirPaths(entityName, record, dirName) {
   return { dir: dirName, filename: finalName, metaPath };
 }
 
+/**
+ * Resolve companion filename collisions within a shared BinID directory.
+ *
+ * Mutates the `filename` field in each entry to apply collision suffixes:
+ *   - Content vs media same name → media gets "(media)" before extension
+ *   - Same entity type duplicates → 2nd+ get "-1", "-2", ... suffix
+ *
+ * Companion files never contain ~UID. The metadata @reference stores the
+ * collision-suffixed name, so dbo rm/push/diff find metadata via @reference
+ * scan rather than filename derivation.
+ *
+ * @param {Array<{entity: string, uid: string, filename: string, dir: string}>} entries
+ */
+export function resolveFilenameCollisions(entries) {
+  // Group by dir + filename
+  const byPath = new Map();
+  for (const entry of entries) {
+    const key = `${entry.dir}/${entry.filename}`;
+    if (!byPath.has(key)) byPath.set(key, []);
+    byPath.get(key).push(entry);
+  }
+
+  for (const group of byPath.values()) {
+    if (group.length <= 1) continue;
+
+    const contentGroup = group.filter(e => e.entity === 'content');
+    const mediaGroup = group.filter(e => e.entity === 'media');
+
+    // Content wins: media gets (media) suffix
+    if (contentGroup.length > 0 && mediaGroup.length > 0) {
+      for (const m of mediaGroup) {
+        const ext = extname(m.filename);
+        const base = basename(m.filename, ext);
+        m.filename = `${base}(media)${ext}`;
+      }
+    }
+
+    // Same-entity duplicates: -1, -2, ... suffix (by insertion order)
+    for (const sameType of [contentGroup, mediaGroup]) {
+      if (sameType.length <= 1) continue;
+      for (let i = 1; i < sameType.length; i++) {
+        const ext = extname(sameType[i].filename);
+        const base = basename(sameType[i].filename, ext);
+        sameType[i].filename = `${base}-${i}${ext}`;
+      }
+    }
+  }
+}
+
+/**
+ * If companion files on disk still use legacy ~UID naming,
+ * rename them to the natural filename and update @references in metadata.
+ *
+ * Handles two cases:
+ *   A) @reference itself contains ~UID (e.g. "@colors~uid.css") → strip ~UID,
+ *      rename file, update @reference in metadata, rewrite metadata file.
+ *   B) @reference is already natural but file on disk has ~UID → just rename file.
+ *
+ * Returns true if any metadata @references were updated (caller should NOT
+ * overwrite metaPath after this — it's already been rewritten).
+ */
+async function detectAndRenameLegacyCompanions(metaPath, meta) {
+  const uid = meta.UID;
+  if (!uid) return false;
+
+  const metaDir = dirname(metaPath);
+  const contentCols = [...(meta._contentColumns || [])];
+  if (meta._mediaFile) contentCols.push('_mediaFile');
+  let metaChanged = false;
+
+  for (const col of contentCols) {
+    const ref = meta[col];
+    if (!ref || !String(ref).startsWith('@')) continue;
+
+    const refName = String(ref).substring(1);
+    // Resolve paths: @/ references are root-relative, others are metaDir-relative
+    const resolveRef = (name) => name.startsWith('/')
+      ? join(process.cwd(), name)
+      : join(metaDir, name);
+
+    // Case A: @reference itself contains ~UID — strip it
+    if (hasUidInFilename(refName, uid)) {
+      const naturalName = stripUidFromFilename(refName, uid);
+      const legacyPath = resolveRef(refName);
+      const naturalPath = resolveRef(naturalName);
+      const legacyExists = await fileExists(legacyPath);
+      const naturalExists = await fileExists(naturalPath);
+
+      if (legacyExists && !naturalExists) {
+        // Rename legacy → natural
+        try {
+          await mkdir(dirname(naturalPath), { recursive: true });
+          await rename(legacyPath, naturalPath);
+          log.dim(`  Legacy companion renamed: ${basename(legacyPath)} → ${basename(naturalPath)}`);
+        } catch { /* rename failed */ }
+      } else if (legacyExists && naturalExists) {
+        // Both exist (clone downloaded fresh copy) — move orphaned legacy file to trash
+        try {
+          const trashDir = join(process.cwd(), 'trash');
+          await mkdir(trashDir, { recursive: true });
+          await rename(legacyPath, join(trashDir, basename(legacyPath)));
+          await ensureTrashIcon(trashDir);
+          log.dim(`  Trashed orphan legacy file: ${basename(legacyPath)}`);
+        } catch { /* non-critical */ }
+      }
+
+      // Update @reference regardless (even if file rename failed, fix the reference)
+      meta[col] = `@${naturalName}`;
+      metaChanged = true;
+      continue;
+    }
+
+    // Case B: @reference is natural but file on disk might still have ~UID
+    const naturalPath = resolveRef(refName);
+    if (await fileExists(naturalPath)) continue;
+
+    const ext = extname(refName);
+    const base = basename(refName, ext);
+    const legacyName = ext ? `${base}~${uid}${ext}` : `${base}~${uid}`;
+    const legacyPath = resolveRef(legacyName);
+
+    if (await fileExists(legacyPath)) {
+      try {
+        await mkdir(dirname(naturalPath), { recursive: true });
+        await rename(legacyPath, naturalPath);
+        log.dim(`  Legacy companion renamed: ${basename(legacyPath)} → ${basename(naturalPath)}`);
+      } catch { /* rename failed */ }
+    }
+  }
+
+  // Rewrite metadata file if @references were updated
+  if (metaChanged) {
+    try {
+      await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+    } catch { /* non-critical */ }
+  }
+
+  return metaChanged;
+}
+
+/**
+ * Scan all directories under Bins/ for orphaned legacy ~UID companion files
+ * that no metadata @reference points to, and move them to trash/.
+ *
+ * A file is considered an orphan if:
+ *   - It contains ~ in its name (potential legacy ~UID naming)
+ *   - It's NOT a .metadata.json file
+ *   - No .metadata.json in the same directory has an @reference pointing to it
+ */
+async function trashOrphanedLegacyCompanions() {
+  const binsDir = join(process.cwd(), BINS_DIR);
+  if (!await fileExists(binsDir)) return;
+
+  // Collect all @references from all metadata files, and all non-metadata files with ~
+  const referencedFiles = new Set(); // Set of absolute paths referenced by metadata
+  const tildeFiles = []; // non-metadata files containing ~
+
+  async function scan(dir) {
+    let entries;
+    try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; }
+
+    for (const entry of entries) {
+      if (entry.name.startsWith('.')) continue;
+      const full = join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        await scan(full);
+        continue;
+      }
+
+      if (entry.name.endsWith('.metadata.json')) {
+        // Read metadata and collect all @references
+        try {
+          const meta = JSON.parse(await readFile(full, 'utf8'));
+          const cols = [...(meta._contentColumns || [])];
+          if (meta._mediaFile) cols.push('_mediaFile');
+          for (const col of cols) {
+            const ref = meta[col];
+            if (ref && String(ref).startsWith('@')) {
+              const refName = String(ref).substring(1);
+              // Handle both relative and @/ (root-relative) references
+              if (refName.startsWith('/')) {
+                referencedFiles.add(join(process.cwd(), refName));
+              } else {
+                referencedFiles.add(join(dir, refName));
+              }
+            }
+          }
+        } catch { /* skip unreadable metadata */ }
+      } else if (entry.name.includes('~')) {
+        tildeFiles.push(full);
+      }
+    }
+  }
+
+  await scan(binsDir);
+
+  // Filter to orphans: ~ files not referenced by any metadata
+  const orphans = tildeFiles.filter(f => !referencedFiles.has(f));
+  if (orphans.length === 0) return;
+
+  const trashDir = join(process.cwd(), 'trash');
+  await mkdir(trashDir, { recursive: true });
+  let trashed = 0;
+
+  for (const orphan of orphans) {
+    try {
+      await rename(orphan, join(trashDir, basename(orphan)));
+      trashed++;
+    } catch { /* non-critical */ }
+  }
+
+  if (trashed > 0) {
+    await ensureTrashIcon(trashDir);
+    log.dim(`  Trashed ${trashed} orphaned legacy ~UID companion file(s)`);
+  }
+}
+
 /**
  * Build global registry of all files that will be created during clone.
  * Returns Map<filePath, Array<{entity, record, dir, filename, metaPath}>>
@@ -204,25 +568,34 @@ export function resolveEntityDirPaths(entityName, record, dirName) {
 async function buildFileRegistry(appJson, structure, placementPrefs) {
   const registry = new Map();
 
-  function addToRegistry(filePath, entity, record, dir, filename, metaPath) {
-    if (!registry.has(filePath)) {
-      registry.set(filePath, []);
-    }
-    registry.get(filePath).push({ entity, record, dir, filename, metaPath });
-  }
+  // Collect all entries first for collision resolution
+  const allEntries = [];
 
   // Process content records
   for (const record of (appJson.children.content || [])) {
     const { dir, filename, metaPath } = resolveRecordPaths(
       'content', record, structure, placementPrefs.contentPlacement
     );
-    addToRegistry(join(dir, filename), 'content', record, dir, filename, metaPath);
+    allEntries.push({ entity: 'content', uid: record.UID, record, dir, filename, metaPath });
   }
 
   // Process media records
   for (const record of (appJson.children.media || [])) {
     const { dir, filename, metaPath } = resolveMediaPaths(record, structure);
-    addToRegistry(join(dir, filename), 'media', record, dir, filename, metaPath);
+    allEntries.push({ entity: 'media', uid: record.UID, record, dir, filename, metaPath });
+  }
+
+  // Auto-resolve content vs media collisions and same-entity duplicates
+  // (content wins, media gets "(media)" suffix; same-entity duplicates get "-N" suffix)
+  resolveFilenameCollisions(allEntries);
+
+  // Build registry with resolved filenames
+  for (const entry of allEntries) {
+    const filePath = join(entry.dir, entry.filename);
+    if (!registry.has(filePath)) {
+      registry.set(filePath, []);
+    }
+    registry.get(filePath).push(entry);
   }
 
   // Note: entity-dir records (Extensions/, Data Sources/, etc.) and generic entities
@@ -803,6 +1176,7 @@ export async function performClone(source, options = {}) {
 
   // Step 4c: Detect and resolve file path collisions (skip in pull mode and entity-filter mode)
   let toDeleteUIDs = new Set();
+  let resolvedFilenames = new Map(); // UID → resolved filename (after collision resolution)
   if (!options.pullMode && !entityFilter) {
     log.info('Scanning for file path collisions...');
     const fileRegistry = await buildFileRegistry(appJson, structure, placementPrefs);
@@ -811,6 +1185,15 @@ export async function performClone(source, options = {}) {
     if (toDeleteUIDs.size > 0) {
       await stageCollisionDeletions(toDeleteUIDs, appJson, options);
     }
+
+    // Build UID → filename map from the collision-resolved registry
+    for (const entries of fileRegistry.values()) {
+      for (const entry of entries) {
+        if (entry.record.UID) {
+          resolvedFilenames.set(entry.record.UID, entry.filename);
+        }
+      }
+    }
   }
 
   // Pre-load previous baseline for fast _LastUpdated string comparison in isServerNewer.
@@ -827,6 +1210,7 @@ export async function performClone(source, options = {}) {
       placementPrefs.contentPlacement,
       serverTz,
       toDeleteUIDs,
+      resolvedFilenames,
     );
   }
 
@@ -840,7 +1224,7 @@ export async function performClone(source, options = {}) {
   if (!entityFilter || entityFilter.has('media')) {
     const mediaEntries = appJson.children.media || [];
     if (mediaEntries.length > 0) {
-      mediaRefs = await processMediaEntries(mediaEntries, structure, options, config, appJson.ShortName, serverTz, toDeleteUIDs);
+      mediaRefs = await processMediaEntries(mediaEntries, structure, options, config, appJson.ShortName, serverTz, toDeleteUIDs, resolvedFilenames);
     }
   }
 
@@ -894,6 +1278,16 @@ export async function performClone(source, options = {}) {
     resetBaselineCache(); // invalidate so next operation reloads the fresh baseline
   }
 
+  // Step 8.5: Detect and trash orphaned local records (deleted on server)
+  if (!entityFilter) {
+    const ig = await loadIgnore();
+    const sync = await loadSynchronize();
+    await detectAndTrashOrphans(appJson, ig, sync, { ...options, entityFilter });
+  }
+
+  // Step 9: Trash orphaned legacy ~UID companion files that no metadata references
+  await trashOrphanedLegacyCompanions();
+
   log.plain('');
   const verb = options.pullMode ? 'Pull' : 'Clone';
   log.success(entityFilter ? `${verb} complete! (filtered: ${options.entity})` : `${verb} complete!`);
@@ -1133,7 +1527,7 @@ async function updatePackageJson(appJson, config) {
  * Process content entries: write files + metadata, return reference map.
  * Returns array of { uid, metaPath } for app.json reference replacement.
  */
-async function processContentEntries(contents, structure, options, contentPlacement, serverTz, skipUIDs = new Set()) {
+async function processContentEntries(contents, structure, options, contentPlacement, serverTz, skipUIDs = new Set(), resolvedFilenames = new Map()) {
   if (!contents || contents.length === 0) return [];
 
   const refs = [];
@@ -1151,7 +1545,8 @@ async function processContentEntries(contents, structure, options, contentPlacem
       log.dim(`  Skipped ${record.Name || record.UID} (collision rejection)`);
       continue;
     }
-    const ref = await processRecord('content', record, structure, options, usedNames, placementPreference, serverTz, bulkAction);
+    const filenameOverride = resolvedFilenames.get(record.UID) || null;
+    const ref = await processRecord('content', record, structure, options, usedNames, placementPreference, serverTz, bulkAction, filenameOverride);
     if (ref) refs.push(ref);
   }
 
@@ -1378,33 +1773,47 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
       if (bulkAction.value !== 'overwrite_all') {
         const configWithTz = { ...config, ServerTimezone: serverTz };
         const localSyncTime = await getLocalSyncTime(metaPath);
-        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz, 'content', record.UID);
+
+        // If local metadata has no _LastUpdated (e.g. from dbo add), treat as server-newer
+        let localMissingLastUpdated = false;
+        try {
+          const localMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+          if (!localMeta._LastUpdated) localMissingLastUpdated = true;
+        } catch { /* unreadable */ }
+
+        const serverNewer = localMissingLastUpdated || isServerNewer(localSyncTime, record._LastUpdated, configWithTz, 'content', record.UID);
         const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
         if (serverNewer) {
-          const action = await promptChangeDetection(finalName, record, configWithTz, {
-            serverDate,
-            localDate: localSyncTime,
-          });
+          // Incomplete metadata (no _LastUpdated) from dbo add — auto-accept without prompting
+          if (localMissingLastUpdated) {
+            log.dim(`  Completing metadata: ${finalName}`);
+            // Fall through to write
+          } else {
+            const action = await promptChangeDetection(finalName, record, configWithTz, {
+              serverDate,
+              localDate: localSyncTime,
+            });
 
-          if (action === 'skip') {
-            log.dim(`  Skipped ${finalName}`);
-            refs.push({ uid: record.UID, metaPath });
-            continue;
-          }
-          if (action === 'skip_all') {
-            bulkAction.value = 'skip_all';
-            log.dim(`  Skipped ${finalName}`);
-            refs.push({ uid: record.UID, metaPath });
-            continue;
-          }
-          if (action === 'overwrite_all') {
-            bulkAction.value = 'overwrite_all';
-          }
-          if (action === 'compare') {
-            await inlineDiffAndMerge(record, metaPath, configWithTz);
-            refs.push({ uid: record.UID, metaPath });
-            continue;
+            if (action === 'skip') {
+              log.dim(`  Skipped ${finalName}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'skip_all') {
+              bulkAction.value = 'skip_all';
+              log.dim(`  Skipped ${finalName}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'overwrite_all') {
+              bulkAction.value = 'overwrite_all';
+            }
+            if (action === 'compare') {
+              await inlineDiffAndMerge(record, metaPath, configWithTz);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
           }
         } else {
           const locallyModified = await hasLocalModifications(metaPath, configWithTz);
@@ -1452,25 +1861,24 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
 
       // Check if this column should be extracted as a companion file
       const extractInfo = contentColsToExtract.find(c => c.col === key);
-      if (extractInfo && value && typeof value === 'object' && value.encoding === 'base64' && value.value !== null) {
-        const decoded = resolveContentValue(value);
-        if (decoded) {
-          const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
-          const colFilePath = join(dirName, colFileName);
-          await writeFile(colFilePath, decoded);
-          meta[key] = `@${colFileName}`;
-          extractedContentCols.push(key);
-
-          // Set timestamps on companion file
-          if (serverTz && (record._CreatedOn || record._LastUpdated)) {
-            try {
-              await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz);
-            } catch { /* non-critical */ }
-          }
+      if (extractInfo) {
+        const isBase64 = value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64';
+        const decoded = isBase64 ? (resolveContentValue(value) ?? '') : (value ?? '');
+        const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
+        const colFilePath = join(dirName, colFileName);
+        await writeFile(colFilePath, decoded);
+        meta[key] = `@${colFileName}`;
+        extractedContentCols.push(key);
 
-          log.dim(`  → ${colFilePath}`);
-          continue;
+        // Set timestamps on companion file
+        if (serverTz && (record._CreatedOn || record._LastUpdated)) {
+          try {
+            await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz);
+          } catch { /* non-critical */ }
         }
+
+        log.dim(`  → ${colFilePath}`);
+        continue;
       }
 
       // Other base64 columns not selected for extraction — decode inline
@@ -1841,8 +2249,34 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
         }
       }
 
+      // Rename legacy ~UID companion files and update @references in extension metadata
+      if (await fileExists(metaPath)) {
+        try {
+          const existingMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+          await detectAndRenameLegacyCompanions(metaPath, existingMeta);
+        } catch { /* non-critical */ }
+      }
+
+      // Check if any @reference content files are missing — force re-extraction if so
+      let hasNewExtractions = contentColsToExtract.length > 0;
+      if (!hasNewExtractions && await fileExists(metaPath)) {
+        try {
+          const existingMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+          for (const col of (existingMeta._contentColumns || [])) {
+            const ref = existingMeta[col];
+            if (ref && String(ref).startsWith('@')) {
+              const refName = String(ref).substring(1);
+              const refPath = refName.startsWith('/') ? join(process.cwd(), refName) : join(dir, refName);
+              if (!await fileExists(refPath)) {
+                hasNewExtractions = true; // Force re-extraction
+                break;
+              }
+            }
+          }
+        } catch { /* non-critical */ }
+      }
+
       // Change detection — same pattern as processEntityDirEntries()
-      const hasNewExtractions = contentColsToExtract.length > 0;
       // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
       const willDeleteExtMeta = join(dir, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
       const extMetaExists = await fileExists(metaPath) && !await fileExists(willDeleteExtMeta);
@@ -1889,32 +2323,30 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
         if (key === 'children') continue;
 
         const extractInfo = contentColsToExtract.find(c => c.col === key);
-        if (extractInfo && value && typeof value === 'object'
-            && value.encoding === 'base64' && value.value !== null) {
-          const decoded = resolveContentValue(value);
-          if (decoded) {
-            let colFilePath, refValue;
-
-            if (mdColInfo && extractInfo.col === mdColInfo.col) {
-              // Root placement: docs/<name>~<uid>.md
-              const docFileName = `${finalName}.md`;
-              colFilePath = join(DOCUMENTATION_DIR, docFileName);
-              refValue = `@/${DOCUMENTATION_DIR}/${docFileName}`;
-            } else {
-              const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
-              colFilePath = join(dir, colFileName);
-              refValue = `@${colFileName}`;
-            }
+        if (extractInfo) {
+          const isBase64 = value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64';
+          const decoded = isBase64 ? (resolveContentValue(value) ?? '') : (value ?? '');
+          let colFilePath, refValue;
+
+          if (mdColInfo && extractInfo.col === mdColInfo.col) {
+            // Root placement: docs/<name>.md (natural name, no ~UID)
+            const docFileName = `${name}.md`;
+            colFilePath = join(DOCUMENTATION_DIR, docFileName);
+            refValue = `@/${DOCUMENTATION_DIR}/${docFileName}`;
+          } else {
+            const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
+            colFilePath = join(dir, colFileName);
+            refValue = `@${colFileName}`;
+          }
 
-            meta[key] = refValue;
-            await writeFile(colFilePath, decoded);
-            extractedCols.push(key);
-            if (serverTz) {
-              try { await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz); } catch {}
-            }
-            log.dim(`  → ${colFilePath}`);
-            continue;
+          meta[key] = refValue;
+          await writeFile(colFilePath, decoded);
+          extractedCols.push(key);
+          if (serverTz) {
+            try { await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz); } catch {}
           }
+          log.dim(`  → ${colFilePath}`);
+          continue;
         }
 
         // Inline or non-extraction columns
@@ -1959,7 +2391,7 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
  * Process media entries: download binary files from server + create metadata.
  * Media uses Filename (not Name) and files are fetched via /api/media/{uid}.
  */
-async function processMediaEntries(mediaRecords, structure, options, config, appShortName, serverTz, skipUIDs = new Set()) {
+async function processMediaEntries(mediaRecords, structure, options, config, appShortName, serverTz, skipUIDs = new Set(), resolvedFilenames = new Map()) {
   if (!mediaRecords || mediaRecords.length === 0) return [];
 
   // Track stale records (404s) for cleanup prompt
@@ -2062,16 +2494,25 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     if (!dir) dir = BINS_DIR;
     await mkdir(dir, { recursive: true });
 
-    // Always include UID in filename via tilde convention
+    // Companion: natural name, no UID (use collision-resolved override if available)
     const uid = String(record.UID || record._id || 'untitled');
-    const base = buildUidFilename(name, uid);
-    const finalFilename = `${base}.${ext}`;
+    const finalFilename = resolvedFilenames.get(record.UID) || sanitizeFilename(filename);
     const filePath = join(dir, finalFilename);
-    const metaPath = join(dir, `${finalFilename}.metadata.json`);
+    // Metadata: name~uid.ext.metadata.json (unchanged format)
+    const metaBase = buildUidFilename(name, uid);
+    const metaPath = join(dir, `${metaBase}.${ext}.metadata.json`);
     // usedNames retained for tracking
     const fileKey = `${dir}/${name}.${ext}`;
     usedNames.set(fileKey, (usedNames.get(fileKey) || 0) + 1);
 
+    // Rename legacy ~UID companion files to natural names if needed
+    if (await fileExists(metaPath)) {
+      try {
+        const existingMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+        await detectAndRenameLegacyCompanions(metaPath, existingMeta);
+      } catch { /* non-critical */ }
+    }
+
     // Change detection for existing media files
     // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
     const willDeleteMediaMeta = join(dir, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
@@ -2093,7 +2534,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
         const diffable = isDiffable(ext);
 
         if (serverNewer) {
-          const action = await promptChangeDetection(dedupName, record, configWithTz, {
+          const action = await promptChangeDetection(finalFilename, record, configWithTz, {
             diffable,
             serverDate,
             localDate: localSyncTime,
@@ -2124,7 +2565,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
           const locallyModified = await hasLocalModifications(metaPath, configWithTz);
           if (locallyModified) {
             const localDate = localSyncTime; // mtime already fetched above
-            const action = await promptChangeDetection(dedupName, record, configWithTz, {
+            const action = await promptChangeDetection(finalFilename, record, configWithTz, {
               localIsNewer: true,
               diffable,
               serverDate,
@@ -2273,7 +2714,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
  * Process a single record: determine directory, write content file + metadata.
  * Returns { uid, metaPath } or null.
  */
-async function processRecord(entityName, record, structure, options, usedNames, _placementPreference, serverTz, bulkAction = { value: null }) {
+async function processRecord(entityName, record, structure, options, usedNames, _placementPreference, serverTz, bulkAction = { value: null }, filenameOverride = null) {
   let name = sanitizeFilename(String(record.Name || record.UID || 'untitled'));
 
   // Determine file extension (priority: Extension field > Name field > Path field > empty)
@@ -2296,7 +2737,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
   }
   // If still no extension, check existing local metadata for a previously chosen extension.
   // On re-clone, the Content @reference in the local metadata already has the extension
-  // the user picked on the first clone (e.g. "@CurrentTask~uid.html" → "html").
+  // the user picked on the first clone (e.g. "@CurrentTask.html" → "html").
   if (!ext && record.UID) {
     try {
       const uid = String(record.UID);
@@ -2398,23 +2839,32 @@ async function processRecord(entityName, record, structure, options, usedNames,
 
   await mkdir(dir, { recursive: true });
 
-  // Always include UID in filename via tilde convention
   const uid = String(record.UID || record._id || 'untitled');
-  const finalName = buildUidFilename(name, uid);
+  // Companion: natural name, no UID (use collision-resolved override if available)
+  const fileName = filenameOverride || sanitizeFilename(buildContentFileName(record, uid));
+  // Metadata: still uses ~UID
+  const metaBase = buildUidFilename(name, uid);
   // usedNames retained for non-UID edge case tracking
   const nameKey = `${dir}/${name}`;
   usedNames.set(nameKey, (usedNames.get(nameKey) || 0) + 1);
 
-  // Write content file if Content column has data
+  // Write content file — always create companion for content entities even if empty
   const contentValue = record.Content;
-  const hasContent = contentValue && (
-    (typeof contentValue === 'object' && contentValue.value) ||
-    (typeof contentValue === 'string' && contentValue.length > 0)
+  const hasContent = contentValue !== null && contentValue !== undefined && (
+    (typeof contentValue === 'object' && contentValue.encoding === 'base64') ||
+    (typeof contentValue === 'string')
   );
 
-  const fileName = ext ? `${finalName}.${ext}` : finalName;
   const filePath = join(dir, fileName);
-  const metaPath = join(dir, `${finalName}.metadata.json`);
+  const metaPath = join(dir, `${metaBase}.metadata.json`);
+
+  // Rename legacy ~UID companion files to natural names if needed
+  if (await fileExists(metaPath)) {
+    try {
+      const existingMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+      await detectAndRenameLegacyCompanions(metaPath, existingMeta);
+    } catch { /* non-critical */ }
+  }
 
   // Change detection: check if file already exists locally
   // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
@@ -2422,7 +2872,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
   const metaExistsForChangeDetect = await fileExists(metaPath) && !await fileExists(willDeleteMeta);
   if (metaExistsForChangeDetect && !options.yes) {
     if (bulkAction.value === 'skip_all') {
-      log.dim(`  Skipped ${finalName}.${ext}`);
+      log.dim(`  Skipped ${fileName}`);
       return { uid: record.UID, metaPath };
     }
 
@@ -2430,51 +2880,66 @@ async function processRecord(entityName, record, structure, options, usedNames,
       const config = await loadConfig();
       const configWithTz = { ...config, ServerTimezone: serverTz };
       const localSyncTime = await getLocalSyncTime(metaPath);
-      const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz, entityName, record.UID);
+
+      // If local metadata has no _LastUpdated (e.g. from dbo add with incomplete fields),
+      // always treat as server-newer so pull populates missing columns.
+      let localMissingLastUpdated = false;
+      try {
+        const localMeta = JSON.parse(await readFile(metaPath, 'utf8'));
+        if (!localMeta._LastUpdated) localMissingLastUpdated = true;
+      } catch { /* unreadable — will be overwritten */ }
+
+      const serverNewer = localMissingLastUpdated || isServerNewer(localSyncTime, record._LastUpdated, configWithTz, entityName, record.UID);
       const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
       if (serverNewer) {
-        const action = await promptChangeDetection(finalName, record, configWithTz, {
-          serverDate,
-          localDate: localSyncTime,
-        });
-
-        if (action === 'skip') {
-          log.dim(`  Skipped ${finalName}.${ext}`);
-          return { uid: record.UID, metaPath };
-        }
-        if (action === 'skip_all') {
-          bulkAction.value = 'skip_all';
-          log.dim(`  Skipped ${finalName}.${ext}`);
-          return { uid: record.UID, metaPath };
-        }
-        if (action === 'overwrite_all') {
-          bulkAction.value = 'overwrite_all';
+        // Incomplete metadata (no _LastUpdated) from dbo add — auto-accept without prompting
+        if (localMissingLastUpdated) {
+          log.dim(`  Completing metadata: ${fileName}`);
           // Fall through to write
+        } else {
+          const action = await promptChangeDetection(fileName, record, configWithTz, {
+            serverDate,
+            localDate: localSyncTime,
+          });
+
+          if (action === 'skip') {
+            log.dim(`  Skipped ${fileName}`);
+            return { uid: record.UID, metaPath };
+          }
+          if (action === 'skip_all') {
+            bulkAction.value = 'skip_all';
+            log.dim(`  Skipped ${fileName}`);
+            return { uid: record.UID, metaPath };
+          }
+          if (action === 'overwrite_all') {
+            bulkAction.value = 'overwrite_all';
+            // Fall through to write
+          }
+          if (action === 'compare') {
+            await inlineDiffAndMerge(record, metaPath, configWithTz);
+            return { uid: record.UID, metaPath };
+          }
+          // 'overwrite' falls through to normal write
         }
-        if (action === 'compare') {
-          await inlineDiffAndMerge(record, metaPath, configWithTz);
-          return { uid: record.UID, metaPath };
-        }
-        // 'overwrite' falls through to normal write
       } else {
         // Server _LastUpdated hasn't changed since last sync.
         // Check if local content files were modified (user edits).
         const locallyModified = await hasLocalModifications(metaPath, configWithTz);
         if (locallyModified) {
-          const action = await promptChangeDetection(finalName, record, configWithTz, {
+          const action = await promptChangeDetection(fileName, record, configWithTz, {
             localIsNewer: true,
             serverDate,
             localDate: localSyncTime,
           });
 
           if (action === 'skip') {
-            log.dim(`  Kept local: ${finalName}.${ext}`);
+            log.dim(`  Kept local: ${fileName}`);
             return { uid: record.UID, metaPath };
           }
           if (action === 'skip_all') {
             bulkAction.value = 'skip_all';
-            log.dim(`  Kept local: ${finalName}.${ext}`);
+            log.dim(`  Kept local: ${fileName}`);
             return { uid: record.UID, metaPath };
           }
           if (action === 'overwrite_all') {
@@ -2486,7 +2951,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
           }
           // 'overwrite' falls through to normal write
         } else {
-          log.dim(`  Up to date: ${finalName}.${ext}`);
+          log.dim(`  Up to date: ${fileName}`);
           return { uid: record.UID, metaPath };
         }
       }
@@ -2494,11 +2959,9 @@ async function processRecord(entityName, record, structure, options, usedNames,
   }
 
   if (hasContent) {
-    const decoded = resolveContentValue(contentValue);
-    if (decoded) {
-      await writeFile(filePath, decoded);
-      log.success(`Saved ${filePath}`);
-    }
+    const decoded = resolveContentValue(contentValue) ?? '';
+    await writeFile(filePath, decoded);
+    log.success(`Saved ${filePath}`);
   }
 
   // Build metadata
@@ -2515,7 +2978,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
       if (decoded && decoded.length > 200) {
         // Large value: save as separate file
         const colExt = guessExtensionForColumn(key);
-        const colFileName = `${finalName}-${key.toLowerCase()}.${colExt}`;
+        const colFileName = `${metaBase}-${key.toLowerCase()}.${colExt}`;
         const colFilePath = join(dir, colFileName);
         await writeFile(colFilePath, decoded);
         meta[key] = `@${colFileName}`;