@dboio/cli 0.8.0 → 0.9.2

package/src/commands/clone.js

@@ -3,11 +3,13 @@ import { readFile, writeFile, mkdir, access } from 'fs/promises';
 import { join, basename, extname } from 'path';
 import { DboClient } from '../lib/client.js';
 import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference, saveCloneSource, loadCloneSource, saveDescriptorFilenamePreference, loadDescriptorFilenamePreference, saveDescriptorContentExtractions, loadDescriptorContentExtractions, saveExtensionDocumentationMDPlacement, loadExtensionDocumentationMDPlacement } from '../lib/config.js';
-import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_MAP, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir } from '../lib/structure.js';
+import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_NAMES, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
+import { buildUidFilename, detectLegacyDotUid } from '../lib/filenames.js';
 import { setFileTimestamps, parseServerDate } from '../lib/timestamps.js';
 import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge, isDiffable } from '../lib/diff.js';
 import { checkDomainChange } from '../lib/domain-guard.js';
+import { loadMetadataTemplates, saveMetadataTemplates, getTemplateCols, setTemplateCols, buildTemplateFromCloneRecord } from '../lib/metadata-templates.js';
 
 /**
  * Resolve a column value that may be base64-encoded.
@@ -32,6 +34,12 @@ async function fileExists(path) {
   try { await access(path); return true; } catch { return false; }
 }
 
+const WILL_DELETE_PREFIX = '__WILL_DELETE__';
+
+function isWillDeleteFile(filename) {
+  return basename(filename).startsWith(WILL_DELETE_PREFIX);
+}
+
 /**
  * Resolve a content Path to a directory under Bins/.
  *
@@ -132,8 +140,10 @@ function resolveRecordPaths(entityName, record, structure, placementPref) {
     dir = dir.substring(0, dir.lastIndexOf('/')) || BINS_DIR;
   }
 
-  const filename = ext ? `${name}.${ext}` : name;
-  const metaPath = join(dir, `${name}.metadata.json`);
+  const uid = String(record.UID || record._id || 'untitled');
+  const base = buildUidFilename(name, uid);
+  const filename = ext ? `${base}.${ext}` : base;
+  const metaPath = join(dir, `${base}.metadata.json`);
 
   return { dir, filename, metaPath };
 }
@@ -169,7 +179,9 @@ function resolveMediaPaths(record, structure, placementPref) {
   dir = dir.replace(/^\/+|\/+$/g, '');
   if (!dir) dir = BINS_DIR;
 
-  const finalFilename = `${name}.${ext}`;
+  const uid = String(record.UID || record._id || 'untitled');
+  const base = buildUidFilename(name, uid);
+  const finalFilename = `${base}.${ext}`;
   const metaPath = join(dir, `${finalFilename}.metadata.json`);
 
   return { dir, filename: finalFilename, metaPath };
@@ -190,7 +202,7 @@ function resolveEntityDirPaths(entityName, record, dirName) {
   }
 
   const uid = record.UID || 'untitled';
-  const finalName = name === uid ? uid : `${name}.${uid}`;
+  const finalName = buildUidFilename(name, uid);
   const metaPath = join(dirName, `${finalName}.metadata.json`);
   return { dir: dirName, filename: finalName, metaPath };
 }
@@ -427,6 +439,7 @@ export const cloneCommand = new Command('clone')
   .option('--force', 'Force re-processing of all files, skip change detection')
   .option('--domain <host>', 'Override domain')
   .option('-y, --yes', 'Auto-accept all prompts')
+  .option('--media-placement <placement>', 'Set media placement: fullpath or binpath (default: bin)')
   .option('-v, --verbose', 'Show HTTP request details')
   .action(async (source, options) => {
     try {
@@ -444,10 +457,22 @@ export const cloneCommand = new Command('clone')
 async function resolveAppSource(source, options, config) {
   if (source) {
     if (source.startsWith('http://') || source.startsWith('https://')) {
-      log.info(`Fetching app JSON from ${source}...`);
-      const res = await fetch(source);
-      if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${source}`);
-      return await res.json();
+      const ora = (await import('ora')).default;
+      const spinner = ora(`Fetching app JSON from ${source}...`).start();
+      let res;
+      try {
+        res = await fetch(source);
+      } catch (err) {
+        spinner.fail(`Failed to fetch from ${source}`);
+        throw err;
+      }
+      if (!res.ok) {
+        spinner.fail(`HTTP ${res.status} fetching ${source}`);
+        throw new Error(`HTTP ${res.status} fetching ${source}`);
+      }
+      const json = await res.json();
+      spinner.succeed('Loaded app JSON');
+      return json;
     }
     log.info(`Loading app JSON from ${source}...`);
     const raw = await readFile(source, 'utf8');
@@ -460,10 +485,22 @@ async function resolveAppSource(source, options, config) {
     if (storedSource && storedSource !== 'default') {
       // Stored source is a local file path or URL — reuse it
       if (storedSource.startsWith('http://') || storedSource.startsWith('https://')) {
-        log.info(`Fetching app JSON from ${storedSource} (stored source)...`);
-        const res = await fetch(storedSource);
-        if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${storedSource}`);
-        return await res.json();
+        const ora = (await import('ora')).default;
+        const spinner = ora(`Fetching app JSON from ${storedSource} (stored source)...`).start();
+        let res;
+        try {
+          res = await fetch(storedSource);
+        } catch (err) {
+          spinner.fail(`Failed to fetch from ${storedSource}`);
+          throw err;
+        }
+        if (!res.ok) {
+          spinner.fail(`HTTP ${res.status} fetching ${storedSource}`);
+          throw new Error(`HTTP ${res.status} fetching ${storedSource}`);
+        }
+        const json = await res.json();
+        spinner.succeed('Loaded app JSON');
+        return json;
       }
       if (await fileExists(storedSource)) {
         log.info(`Loading app JSON from ${storedSource} (stored source)...`);
@@ -738,6 +775,29 @@ export async function performClone(source, options = {}) {
   for (const d of DEFAULT_PROJECT_DIRS) log.dim(`  ${d}/`);
   for (const d of createdDirs) log.dim(`  ${d}/`);
 
+  // Warn about legacy mixed-case directories from pre-0.9.1
+  const LEGACY_DIR_MAP = {
+    'Bins':        'bins',
+    'Automations': 'automation',
+    'App Versions': 'app_version',
+    'Documentation': 'docs',
+    'Sites':       'site',
+    'Extensions':  'extension',
+    'Data Sources': 'data_source',
+    'Groups':      'group',
+    'Integrations': 'integration',
+    'Trash':       'trash',
+    'Src':         'src',
+  };
+  for (const [oldName, newName] of Object.entries(LEGACY_DIR_MAP)) {
+    try {
+      await access(join(process.cwd(), oldName));
+      log.warn(`Legacy directory detected: "${oldName}/" — rename it to "${newName}/" for the new convention.`);
+    } catch {
+      // does not exist — no warning needed
+    }
+  }
+
   // Step 4b: Determine placement preferences (from config or prompt)
   const placementPrefs = await resolvePlacementPreferences(appJson, options);
 
@@ -817,7 +877,7 @@ export async function performClone(source, options = {}) {
       if (refs.length > 0) {
         otherRefs[entityName] = refs;
       }
-    } else if (ENTITY_DIR_MAP[entityName]) {
+    } else if (ENTITY_DIR_NAMES.has(entityName)) {
       // Entity types with project directories — process into their directory
       const refs = await processEntityDirEntries(entityName, entries, options, serverTz);
       if (refs.length > 0) {
@@ -887,8 +947,14 @@ async function resolvePlacementPreferences(appJson, options) {
   let contentPlacement = saved.contentPlacement;
   let mediaPlacement = saved.mediaPlacement;
 
+  // --media-placement flag takes precedence over saved config
+  if (options.mediaPlacement) {
+    mediaPlacement = options.mediaPlacement === 'fullpath' ? 'fullpath' : 'bin';
+    await saveClonePlacement({ contentPlacement: contentPlacement || 'bin', mediaPlacement });
+    log.dim(`  MediaPlacement set to "${mediaPlacement}" via flag`);
+  }
+
   const hasContent = (appJson.children.content || []).length > 0;
-  const hasMedia = (appJson.children.media || []).length > 0;
 
   // If -y flag, default to bin placement (no prompts)
   if (options.yes) {
@@ -906,6 +972,7 @@ async function resolvePlacementPreferences(appJson, options) {
   const inquirer = (await import('inquirer')).default;
   const prompts = [];
 
+  // Only prompt for contentPlacement — media placement is NOT prompted interactively
   if (!contentPlacement && hasContent) {
     prompts.push({
       type: 'list',
@@ -919,23 +986,14 @@ async function resolvePlacementPreferences(appJson, options) {
     });
   }
 
-  if (!mediaPlacement && hasMedia) {
-    prompts.push({
-      type: 'list',
-      name: 'mediaPlacement',
-      message: 'How should media files be placed?',
-      choices: [
-        { name: 'Save all in BinID directory', value: 'bin' },
-        { name: 'Save all in their specified FullPath directory', value: 'fullpath' },
-        { name: 'Ask for every file that has both', value: 'ask' },
-      ],
-    });
+  // Media placement: no interactive prompt — default to 'bin'
+  if (!mediaPlacement) {
+    mediaPlacement = 'bin';
   }
 
   if (prompts.length > 0) {
     const answers = await inquirer.prompt(prompts);
     contentPlacement = contentPlacement || answers.contentPlacement || 'bin';
-    mediaPlacement = mediaPlacement || answers.mediaPlacement || 'bin';
   }
 
   // Resolve defaults for any still-unset values
@@ -960,19 +1018,45 @@ async function resolvePlacementPreferences(appJson, options) {
  */
 async function fetchAppFromServer(appShortName, options, config) {
   const client = new DboClient({ domain: options.domain, verbose: options.verbose });
-  log.info(`Fetching app "${appShortName}" from server...`);
 
-  const result = await client.get(`/api/app/object/${appShortName}`);
+  const ora = (await import('ora')).default;
+  const spinner = ora(`Fetching app "${appShortName}" from server...`).start();
+
+  let result;
+  try {
+    result = await client.get(`/api/app/object/${appShortName}`);
+  } catch (err) {
+    spinner.fail(`Failed to fetch app "${appShortName}"`);
+    throw err;
+  }
 
   const data = result.payload || result.data;
-  const rows = Array.isArray(data) ? data : (data?.Rows || data?.rows || []);
 
-  if (rows.length === 0) {
+  // Handle all response shapes:
+  //   1. Array of rows: [{ UID, ShortName, ... }]
+  //   2. Object with Rows key: { Rows: [...] }
+  //   3. Single app object: { UID, ShortName, children, ... }
+  let appRecord;
+  if (Array.isArray(data)) {
+    appRecord = data.length > 0 ? data[0] : null;
+  } else if (data?.Rows?.length > 0) {
+    appRecord = data.Rows[0];
+  } else if (data?.rows?.length > 0) {
+    appRecord = data.rows[0];
+  } else if (data && typeof data === 'object' && (data.UID || data.ShortName)) {
+    // Single app object returned directly as payload
+    appRecord = data;
+  } else {
+    appRecord = null;
+  }
+
+  if (!appRecord) {
+    spinner.fail(`No app found with ShortName "${appShortName}"`);
     throw new Error(`No app found with ShortName "${appShortName}"`);
   }
 
-  log.success(`Found app on server`);
-  return rows[0];
+  spinner.succeed(`Found app on server`);
+  return appRecord;
 }
 
 /**
@@ -1093,13 +1177,14 @@ async function processGenericEntries(entityName, entries, structure, options, co
 async function processEntityDirEntries(entityName, entries, options, serverTz) {
   if (!entries || entries.length === 0) return [];
 
-  const dirName = ENTITY_DIR_MAP[entityName];
-  if (!dirName) return [];
+  const dirName = entityName;
+  if (!ENTITY_DIR_NAMES.has(entityName)) return [];
 
   await mkdir(dirName, { recursive: true });
 
   const refs = [];
   const bulkAction = { value: null };
+  const legacyRenameAction = { value: null }; // 'rename_all' | 'skip_all' | null
   const config = await loadConfig();
 
   // Determine filename column: saved preference, or prompt, or default
@@ -1247,18 +1332,53 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
       name = sanitizeFilename(String(record.UID || 'untitled'));
     }
 
-    // Include UID in filename to ensure uniqueness (multiple records can share the same name)
-    // Convention: <name>.<UID>.metadata.json — unless name === UID, then just <UID>.metadata.json
+    // Include UID in filename via tilde convention to ensure uniqueness
     const uid = record.UID || 'untitled';
-    const finalName = name === uid ? uid : `${name}.${uid}`;
+    const finalName = buildUidFilename(name, uid);
 
     const metaPath = join(dirName, `${finalName}.metadata.json`);
 
+    // Legacy dot-separator detection: rename <name>.<uid>.metadata.json → <name>~<uid>.metadata.json
+    const legacyMetaPath = join(dirName, `${name}.${uid}.metadata.json`);
+    if (!await fileExists(metaPath) && await fileExists(legacyMetaPath)) {
+      if (options.yes || legacyRenameAction.value === 'rename_all') {
+        const { rename: fsRename } = await import('fs/promises');
+        await fsRename(legacyMetaPath, metaPath);
+        log.dim(`  Auto-renamed: ${basename(legacyMetaPath)} → ${basename(metaPath)}`);
+      } else if (legacyRenameAction.value === 'skip_all') {
+        // skip silently
+      } else {
+        const inquirer = (await import('inquirer')).default;
+        const { action } = await inquirer.prompt([{
+          type: 'list',
+          name: 'action',
+          message: `Found legacy filename "${basename(legacyMetaPath)}" — rename to "${basename(metaPath)}"?`,
+          choices: [
+            { name: 'Yes', value: 'rename' },
+            { name: 'Rename all remaining', value: 'rename_all' },
+            { name: 'Skip', value: 'skip' },
+            { name: 'Skip all remaining', value: 'skip_all' },
+          ],
+        }]);
+        if (action === 'rename' || action === 'rename_all') {
+          const { rename: fsRename } = await import('fs/promises');
+          await fsRename(legacyMetaPath, metaPath);
+          log.success(`  Renamed: ${basename(legacyMetaPath)} → ${basename(metaPath)}`);
+          if (action === 'rename_all') legacyRenameAction.value = 'rename_all';
+        } else {
+          if (action === 'skip_all') legacyRenameAction.value = 'skip_all';
+        }
+      }
+    }
+
     // Change detection for existing files
     // Skip change detection when user has selected new content columns to extract —
     // we need to re-process all records to create the companion files
     const hasNewExtractions = contentColsToExtract.length > 0;
-    if (await fileExists(metaPath) && !options.yes && !hasNewExtractions) {
+    // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
+    const willDeleteEntityMeta = join(dirName, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
+    const entityMetaExists = await fileExists(metaPath) && !await fileExists(willDeleteEntityMeta);
+    if (entityMetaExists && !options.yes && !hasNewExtractions) {
       if (bulkAction.value === 'skip_all') {
         log.dim(`  Skipped ${finalName}`);
         refs.push({ uid: record.UID, metaPath });
@@ -1389,6 +1509,21 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
     refs.push({ uid: record.UID, metaPath });
   }
 
+  // --- Seed metadata template if not already present ---
+  if (entries.length > 0) {
+    const templates = await loadMetadataTemplates();
+    if (templates !== null) {
+      const existing = getTemplateCols(templates, entityName, null);
+      if (!existing) {
+        const firstRecord = entries[0];
+        const extractedColNames = contentColsToExtract.map(c => c.col);
+        const cols = buildTemplateFromCloneRecord(firstRecord, extractedColNames);
+        setTemplateCols(templates, entityName, null, cols);
+        await saveMetadataTemplates(templates);
+      }
+    }
+  }
+
   return refs;
 }
 
@@ -1564,8 +1699,8 @@ async function resolveDocumentationPlacement(options) {
     type: 'list', name: 'placement',
     message: 'Where should extracted documentation MD files be placed?',
     choices: [
-      { name: '/Documentation/<filename>.md  — project root (recommended)', value: 'root' },
-      { name: 'Extensions/Documentation/<filename>.md  — inline alongside metadata', value: 'inline' },
+      { name: 'docs/<filename>.md  — project root (recommended)', value: 'root' },
+      { name: 'extension/documentation/<filename>.md  — inline alongside metadata', value: 'inline' },
     ],
     default: 'root',
   }]);
@@ -1644,6 +1779,7 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
   // Step D: Write files, one group at a time
   const refs = [];
   const bulkAction = { value: null };
+  const legacyRenameAction = { value: null }; // 'rename_all' | 'skip_all' | null
   const config = await loadConfig();
 
   for (const [descriptor, { dir, records }] of groups.entries()) {
@@ -1665,12 +1801,48 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
       }
 
       const uid = record.UID || 'untitled';
-      const finalName = name === uid ? uid : `${name}.${uid}`;
+      const finalName = buildUidFilename(name, uid);
       const metaPath = join(dir, `${finalName}.metadata.json`);
 
+      // Legacy dot-separator detection: rename <name>.<uid>.metadata.json → <name>~<uid>.metadata.json
+      const legacyExtMetaPath = join(dir, `${name}.${uid}.metadata.json`);
+      if (!await fileExists(metaPath) && await fileExists(legacyExtMetaPath)) {
+        if (options.yes || legacyRenameAction.value === 'rename_all') {
+          const { rename: fsRename } = await import('fs/promises');
+          await fsRename(legacyExtMetaPath, metaPath);
+          log.dim(`  Auto-renamed: ${basename(legacyExtMetaPath)} → ${basename(metaPath)}`);
+        } else if (legacyRenameAction.value === 'skip_all') {
+          // skip silently
+        } else {
+          const inquirer = (await import('inquirer')).default;
+          const { action } = await inquirer.prompt([{
+            type: 'list',
+            name: 'action',
+            message: `Found legacy filename "${basename(legacyExtMetaPath)}" — rename to "${basename(metaPath)}"?`,
+            choices: [
+              { name: 'Yes', value: 'rename' },
+              { name: 'Rename all remaining', value: 'rename_all' },
+              { name: 'Skip', value: 'skip' },
+              { name: 'Skip all remaining', value: 'skip_all' },
+            ],
+          }]);
+          if (action === 'rename' || action === 'rename_all') {
+            const { rename: fsRename } = await import('fs/promises');
+            await fsRename(legacyExtMetaPath, metaPath);
+            log.success(`  Renamed: ${basename(legacyExtMetaPath)} → ${basename(metaPath)}`);
+            if (action === 'rename_all') legacyRenameAction.value = 'rename_all';
+          } else {
+            if (action === 'skip_all') legacyRenameAction.value = 'skip_all';
+          }
+        }
+      }
+
       // Change detection — same pattern as processEntityDirEntries()
       const hasNewExtractions = contentColsToExtract.length > 0;
-      if (await fileExists(metaPath) && !options.yes && !hasNewExtractions) {
+      // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
+      const willDeleteExtMeta = join(dir, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
+      const extMetaExists = await fileExists(metaPath) && !await fileExists(willDeleteExtMeta);
+      if (extMetaExists && !options.yes && !hasNewExtractions) {
         if (bulkAction.value === 'skip_all') {
           log.dim(`  Skipped ${finalName}`);
           refs.push({ uid: record.UID, metaPath });
@@ -1720,8 +1892,8 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
             let colFilePath, refValue;
 
             if (mdColInfo && extractInfo.col === mdColInfo.col) {
-              // Root placement: Documentation/<name>.md
-              const docFileName = `${name}.md`;
+              // Root placement: docs/<name>~<uid>.md
+              const docFileName = `${finalName}.md`;
               colFilePath = join(DOCUMENTATION_DIR, docFileName);
               refValue = `@/${DOCUMENTATION_DIR}/${docFileName}`;
             } else {
@@ -1759,6 +1931,21 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
       log.success(`Saved ${metaPath}`);
       refs.push({ uid: record.UID, metaPath });
     }
+
+    // --- Seed metadata template for this descriptor ---
+    if (records.length > 0) {
+      const templates = await loadMetadataTemplates();
+      if (templates !== null) {
+        const existing = getTemplateCols(templates, 'extension', descriptor);
+        if (!existing) {
+          const firstRecord = records[0];
+          const extractedCols = contentColsToExtract.map(c => c.col);
+          const cols = buildTemplateFromCloneRecord(firstRecord, extractedCols);
+          setTemplateCols(templates, 'extension', descriptor, cols);
+          await saveMetadataTemplates(templates);
+        }
+      }
+    }
   }
 
   return refs;
@@ -1886,27 +2073,21 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     if (!dir) dir = BINS_DIR;
     await mkdir(dir, { recursive: true });
 
-    // Deduplicate using full filename (name.ext) so different formats don't collide
-    // e.g. KaTeX_SansSerif-Italic.woff and KaTeX_SansSerif-Italic.woff2 are separate files
-    const fileKey = `${dir}/${name}.${ext}`;
-    const fileCount = usedNames.get(fileKey) || 0;
-    usedNames.set(fileKey, fileCount + 1);
-    let dedupName;
-    if (fileCount > 0) {
-      // Duplicate name — include UID for uniqueness
-      const uid = record.UID || 'untitled';
-      dedupName = name === uid ? uid : `${name}.${uid}`;
-    } else {
-      dedupName = name;
-    }
-    const finalFilename = `${dedupName}.${ext}`;
-    // Metadata: use name.ext as base to avoid collisions between formats
-    // e.g. KaTeX_SansSerif-Italic.woff.metadata.json vs KaTeX_SansSerif-Italic.woff2.metadata.json
+    // Always include UID in filename via tilde convention
+    const uid = String(record.UID || record._id || 'untitled');
+    const base = buildUidFilename(name, uid);
+    const finalFilename = `${base}.${ext}`;
     const filePath = join(dir, finalFilename);
     const metaPath = join(dir, `${finalFilename}.metadata.json`);
+    // usedNames retained for tracking
+    const fileKey = `${dir}/${name}.${ext}`;
+    usedNames.set(fileKey, (usedNames.get(fileKey) || 0) + 1);
 
     // Change detection for existing media files
-    if (await fileExists(metaPath) && !options.yes) {
+    // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
+    const willDeleteMediaMeta = join(dir, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
+    const mediaMetaExists = await fileExists(metaPath) && !await fileExists(willDeleteMediaMeta);
+    if (mediaMetaExists && !options.yes) {
       if (mediaBulkAction.value === 'skip_all') {
         log.dim(`  Skipped ${finalFilename}`);
         refs.push({ uid: record.UID, metaPath });
@@ -2114,6 +2295,41 @@ async function processRecord(entityName, record, structure, options, usedNames,
   }
   // If still no extension, ext remains '' (no extension)
 
+  // If no extension determined and Content column has data, prompt user to choose one
+  if (!ext && !options.yes && record.Content) {
+    const cv = record.Content;
+    const hasContentData = cv && (
+      (typeof cv === 'object' && cv.value !== null && cv.value !== undefined) ||
+      (typeof cv === 'string' && cv.length > 0)
+    );
+    if (hasContentData) {
+      // Decode a snippet for preview
+      let snippet = '';
+      try {
+        const decoded = resolveContentValue(cv);
+        if (decoded) {
+          snippet = decoded.substring(0, 80).replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
+          if (decoded.length > 80) snippet += '...';
+        }
+      } catch { /* ignore decode errors */ }
+      const preview = snippet ? ` (${snippet})` : '';
+      const VALID_CONTENT_EXTENSIONS = ['css', 'js', 'html', 'xml', 'txt', 'md', 'cs', 'json', 'sql'];
+      const inquirer = (await import('inquirer')).default;
+      const { chosenExt } = await inquirer.prompt([{
+        type: 'list',
+        name: 'chosenExt',
+        message: `No extension found for "${record.Name || record.UID}". Choose a file extension for the Content:${preview}`,
+        choices: [
+          ...VALID_CONTENT_EXTENSIONS.map(e => ({ name: `.${e}`, value: e })),
+          { name: 'No extension (skip)', value: '' },
+        ],
+      }]);
+      if (chosenExt) {
+        ext = chosenExt;
+      }
+    }
+  }
+
   // Avoid double extension: if name already ends with .ext, strip it
   if (ext) {
     const extWithDot = `.${ext}`;
@@ -2181,18 +2397,12 @@ async function processRecord(entityName, record, structure, options, usedNames,
 
   await mkdir(dir, { recursive: true });
 
-  // Deduplicate filenames — use UID naming when duplicates exist
+  // Always include UID in filename via tilde convention
+  const uid = String(record.UID || record._id || 'untitled');
+  const finalName = buildUidFilename(name, uid);
+  // usedNames retained for non-UID edge case tracking
   const nameKey = `${dir}/${name}`;
-  const count = usedNames.get(nameKey) || 0;
-  usedNames.set(nameKey, count + 1);
-  let finalName;
-  if (count > 0) {
-    // Duplicate name — include UID for uniqueness
-    const uid = record.UID || 'untitled';
-    finalName = name === uid ? uid : `${name}.${uid}`;
-  } else {
-    finalName = name;
-  }
+  usedNames.set(nameKey, (usedNames.get(nameKey) || 0) + 1);
 
   // Write content file if Content column has data
   const contentValue = record.Content;
@@ -2206,7 +2416,10 @@ async function processRecord(entityName, record, structure, options, usedNames,
   const metaPath = join(dir, `${finalName}.metadata.json`);
 
   // Change detection: check if file already exists locally
-  if (await fileExists(metaPath) && !options.yes) {
+  // Skip __WILL_DELETE__-prefixed files — treat as "no existing file"
+  const willDeleteMeta = join(dir, `${WILL_DELETE_PREFIX}${basename(metaPath)}`);
+  const metaExistsForChangeDetect = await fileExists(metaPath) && !await fileExists(willDeleteMeta);
+  if (metaExistsForChangeDetect && !options.yes) {
     if (bulkAction.value === 'skip_all') {
       log.dim(`  Skipped ${finalName}.${ext}`);
       return { uid: record.UID, metaPath };
@@ -2320,6 +2533,13 @@ async function processRecord(entityName, record, structure, options, usedNames,
     meta._contentColumns = ['Content'];
   }
 
+  // If the extension picker chose an extension (record.Extension was null),
+  // set it in metadata only — not in the record — so the baseline preserves
+  // the server's null and push detects the change.
+  if (ext && !record.Extension) {
+    meta.Extension = ext;
+  }
+
   await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
   log.dim(`  → ${metaPath}`);
 
@@ -2360,12 +2580,22 @@ export function guessExtensionForColumn(columnName) {
  */
 export function buildOutputHierarchyTree(appJson) {
   const outputs = appJson.children.output || [];
-  const columns = appJson.children.output_value || [];
-  const filters = appJson.children.output_value_filter || [];
-  const joins = appJson.children.output_value_entity_column_rel || [];
 
   if (outputs.length === 0) return [];
 
+  // Collect columns/filters/joins from both top-level arrays (if present)
+  // and from each output record's nested .children object.
+  const columns = [...(appJson.children.output_value || [])];
+  const filters = [...(appJson.children.output_value_filter || [])];
+  const joins = [...(appJson.children.output_value_entity_column_rel || [])];
+
+  for (const o of outputs) {
+    if (!o.children) continue;
+    if (Array.isArray(o.children.output_value)) columns.push(...o.children.output_value);
+    if (Array.isArray(o.children.output_value_filter)) filters.push(...o.children.output_value_filter);
+    if (Array.isArray(o.children.output_value_entity_column_rel)) joins.push(...o.children.output_value_entity_column_rel);
+  }
+
   // Index all entities by their numeric ID for O(1) lookups
   const outputById = new Map();
   const columnById = new Map();
@@ -2487,7 +2717,17 @@ async function resolveOutputFilenameColumns(appJson, options) {
     }
 
     // Find a sample record to get available columns
-    const records = appJson.children[entityKey] || [];
+    // Check top-level first, then look inside nested output children
+    let records = appJson.children[entityKey] || [];
+    if (records.length === 0 && entityKey !== 'output') {
+      const outputs = appJson.children.output || [];
+      for (const o of outputs) {
+        if (o.children && Array.isArray(o.children[entityKey]) && o.children[entityKey].length > 0) {
+          records = o.children[entityKey];
+          break;
+        }
+      }
+    }
     if (records.length === 0) {
       result[entityKey] = defaults[entityKey];
       continue;
@@ -2811,18 +3051,24 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
     for (const [key, value] of Object.entries(output)) {
       if (key === '_children') continue;
 
-      // Always extract CustomSQL to companion .sql file
+      // Extract CustomSQL to companion .sql file when Type is CustomSQL (even if empty)
+      // or when the column has actual content
       if (key === 'CustomSQL') {
         const decoded = resolveContentValue(value);
-        const sqlContent = (decoded && decoded.trim()) ? decoded : '';
-        const sqlFilePath = rootMetaPath.replace(/\.json$/, '.CustomSQL.sql');
-        await writeFile(sqlFilePath, sqlContent);
-        rootMeta[key] = `@${basename(sqlFilePath)}`;
-        rootContentColumns.push('CustomSQL');
-        if (serverTz && (output._CreatedOn || output._LastUpdated)) {
-          try { await setFileTimestamps(sqlFilePath, output._CreatedOn, output._LastUpdated, serverTz); } catch { /* non-critical */ }
+        const hasContent = decoded && decoded.trim();
+        if (output.Type === 'CustomSQL' || hasContent) {
+          const sqlFilePath = rootMetaPath.replace(/\.json$/, '.CustomSQL.sql');
+          await writeFile(sqlFilePath, hasContent ? decoded : '');
+          rootMeta[key] = `@${basename(sqlFilePath)}`;
+          rootContentColumns.push('CustomSQL');
+          if (serverTz && (output._CreatedOn || output._LastUpdated)) {
+            try { await setFileTimestamps(sqlFilePath, output._CreatedOn, output._LastUpdated, serverTz); } catch { /* non-critical */ }
+          }
+          log.dim(`  → ${sqlFilePath}`);
+          continue;
         }
-        log.dim(`  → ${sqlFilePath}`);
+        // Not CustomSQL type and empty — store inline
+        rootMeta[key] = '';
         continue;
       }
 
@@ -2872,21 +3118,27 @@ async function writeOutputEntityFile(node, physicalEntity, filePath, serverTz) {
   for (const [key, value] of Object.entries(node)) {
     if (key === '_children') continue;
 
-    // Always extract CustomSQL to companion .sql file
+    // Extract CustomSQL to companion .sql file when Type is CustomSQL (even if empty)
+    // or when the column has actual content
     if (key === 'CustomSQL') {
       const decoded = resolveContentValue(value);
-      const sqlContent = (decoded && decoded.trim()) ? decoded : '';
-      const sqlFilePath = filePath.replace(/\.json$/, '.CustomSQL.sql');
-      await writeFile(sqlFilePath, sqlContent);
-      meta[key] = `@${basename(sqlFilePath)}`;
-      contentColumns.push('CustomSQL');
-
-      if (serverTz && (node._CreatedOn || node._LastUpdated)) {
-        try {
-          await setFileTimestamps(sqlFilePath, node._CreatedOn, node._LastUpdated, serverTz);
-        } catch { /* non-critical */ }
+      const hasContent = decoded && decoded.trim();
+      if (node.Type === 'CustomSQL' || hasContent) {
+        const sqlFilePath = filePath.replace(/\.json$/, '.CustomSQL.sql');
+        await writeFile(sqlFilePath, hasContent ? decoded : '');
+        meta[key] = `@${basename(sqlFilePath)}`;
+        contentColumns.push('CustomSQL');
+
+        if (serverTz && (node._CreatedOn || node._LastUpdated)) {
+          try {
+            await setFileTimestamps(sqlFilePath, node._CreatedOn, node._LastUpdated, serverTz);
+          } catch { /* non-critical */ }
+        }
+        log.dim(`  → ${sqlFilePath}`);
+        continue;
       }
-      log.dim(`  → ${sqlFilePath}`);
+      // Not CustomSQL type and empty — store inline
+      meta[key] = '';
       continue;
     }