@dboio/cli 0.7.2 → 0.8.2

package/src/commands/clone.js

@@ -2,11 +2,11 @@ import { Command } from 'commander';
 import { readFile, writeFile, mkdir, access } from 'fs/promises';
 import { join, basename, extname } from 'path';
 import { DboClient } from '../lib/client.js';
-import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference } from '../lib/config.js';
-import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_MAP, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES } from '../lib/structure.js';
+import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference, saveCloneSource, loadCloneSource, saveDescriptorFilenamePreference, loadDescriptorFilenamePreference, saveDescriptorContentExtractions, loadDescriptorContentExtractions, saveExtensionDocumentationMDPlacement, loadExtensionDocumentationMDPlacement } from '../lib/config.js';
+import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_MAP, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
-import { setFileTimestamps } from '../lib/timestamps.js';
-import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge } from '../lib/diff.js';
+import { setFileTimestamps, parseServerDate } from '../lib/timestamps.js';
+import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge, isDiffable } from '../lib/diff.js';
 import { checkDomainChange } from '../lib/domain-guard.js';
 
 /**
@@ -422,6 +422,8 @@ export const cloneCommand = new Command('clone')
   .argument('[source]', 'Local JSON file path (optional)')
   .option('--app <shortName>', 'App short name to fetch from server')
   .option('-e, --entity <type>', 'Only clone a specific entity type (e.g. output, content, media, extension)')
+  .option('--documentation-only', 'When used with -e extension, clone only documentation extensions')
+  .option('--descriptor-types <bool>', 'Sort extensions into descriptor sub-directories (default: true)', 'true')
   .option('--force', 'Force re-processing of all files, skip change detection')
   .option('--domain <host>', 'Override domain')
   .option('-y, --yes', 'Auto-accept all prompts')
@@ -436,26 +438,70 @@ export const cloneCommand = new Command('clone')
   });
 
 /**
- * Main clone workflow. Exported for use by init --clone.
+ * Resolve app JSON from a given source (file path, URL, or server fetch).
+ * Throws on failure so the caller can retry with a different source.
  */
-export async function performClone(source, options = {}) {
-  const config = await loadConfig();
-  const effectiveDomain = options.domain || config.domain;
-  let appJson;
-
-  // Step 1: Load the app JSON
+async function resolveAppSource(source, options, config) {
   if (source) {
-    // Local file
+    if (source.startsWith('http://') || source.startsWith('https://')) {
+      const ora = (await import('ora')).default;
+      const spinner = ora(`Fetching app JSON from ${source}...`).start();
+      let res;
+      try {
+        res = await fetch(source);
+      } catch (err) {
+        spinner.fail(`Failed to fetch from ${source}`);
+        throw err;
+      }
+      if (!res.ok) {
+        spinner.fail(`HTTP ${res.status} fetching ${source}`);
+        throw new Error(`HTTP ${res.status} fetching ${source}`);
+      }
+      const json = await res.json();
+      spinner.succeed('Loaded app JSON');
+      return json;
+    }
     log.info(`Loading app JSON from ${source}...`);
     const raw = await readFile(source, 'utf8');
-    appJson = JSON.parse(raw);
+    return JSON.parse(raw);
   } else if (options.app) {
-    // Fetch from server by AppShortName
-    appJson = await fetchAppFromServer(options.app, options, config);
-  } else if (config.AppShortName) {
-    // Use config's AppShortName
-    appJson = await fetchAppFromServer(config.AppShortName, options, config);
+    return await fetchAppFromServer(options.app, options, config);
   } else {
+    // Try stored cloneSource before falling back to server fetch
+    const storedSource = await loadCloneSource();
+    if (storedSource && storedSource !== 'default') {
+      // Stored source is a local file path or URL — reuse it
+      if (storedSource.startsWith('http://') || storedSource.startsWith('https://')) {
+        const ora = (await import('ora')).default;
+        const spinner = ora(`Fetching app JSON from ${storedSource} (stored source)...`).start();
+        let res;
+        try {
+          res = await fetch(storedSource);
+        } catch (err) {
+          spinner.fail(`Failed to fetch from ${storedSource}`);
+          throw err;
+        }
+        if (!res.ok) {
+          spinner.fail(`HTTP ${res.status} fetching ${storedSource}`);
+          throw new Error(`HTTP ${res.status} fetching ${storedSource}`);
+        }
+        const json = await res.json();
+        spinner.succeed('Loaded app JSON');
+        return json;
+      }
+      if (await fileExists(storedSource)) {
+        log.info(`Loading app JSON from ${storedSource} (stored source)...`);
+        const raw = await readFile(storedSource, 'utf8');
+        return JSON.parse(raw);
+      }
+      // Stored file no longer exists — fall through to server fetch
+      log.dim(`  Stored clone source "${storedSource}" not found, trying server...`);
+    }
+
+    if (config.AppShortName) {
+      return await fetchAppFromServer(config.AppShortName, options, config);
+    }
+
     // Prompt
     const inquirer = (await import('inquirer')).default;
     const { choice } = await inquirer.prompt([{
@@ -474,7 +520,7 @@ export async function performClone(source, options = {}) {
         message: 'App short name:',
         validate: v => v.trim() ? true : 'App short name is required',
       }]);
-      appJson = await fetchAppFromServer(appName, options, config);
+      return await fetchAppFromServer(appName, options, config);
     } else {
       const { filePath } = await inquirer.prompt([{
         type: 'input', name: 'filePath',
@@ -482,7 +528,155 @@ export async function performClone(source, options = {}) {
         validate: v => v.trim() ? true : 'File path is required',
       }]);
       const raw = await readFile(filePath, 'utf8');
-      appJson = JSON.parse(raw);
+      return JSON.parse(raw);
+    }
+  }
+}
+
+/**
+ * Check synchronize.json for un-pushed staged items before cloning.
+ * Warns the user and offers options: push first, review individually, ignore, or clear.
+ */
+async function checkPendingSynchronize(options) {
+  const sync = await loadSynchronize();
+  const deleteCount = (sync.delete || []).length;
+  const editCount = (sync.edit || []).length;
+  const addCount = (sync.add || []).length;
+  const totalCount = deleteCount + editCount + addCount;
+
+  if (totalCount === 0) return;
+
+  log.warn('');
+  log.warn(`  ⚠  There are ${totalCount} un-pushed staged item(s) in .dbo/synchronize.json:`);
+  if (deleteCount > 0) log.warn(`     ${deleteCount} pending deletion(s)`);
+  if (editCount > 0) log.warn(`     ${editCount} pending edit(s)`);
+  if (addCount > 0) log.warn(`     ${addCount} pending add(s)`);
+  log.warn('');
+
+  if (options.yes) {
+    log.dim('  Ignoring staged items (-y flag)');
+    return;
+  }
+
+  const inquirer = (await import('inquirer')).default;
+  const { action } = await inquirer.prompt([{
+    type: 'list',
+    name: 'action',
+    message: 'How would you like to handle the staged items?',
+    choices: [
+      { name: 'Ignore — keep staged items and continue cloning', value: 'ignore' },
+      { name: 'Review — show each item and decide individually', value: 'review' },
+      { name: 'Clear — discard all staged items and continue cloning', value: 'clear' },
+      { name: 'Abort — stop cloning so you can push first', value: 'abort' },
+    ],
+  }]);
+
+  if (action === 'abort') {
+    log.info('Clone aborted. Run "dbo push" to process staged items first.');
+    process.exit(0);
+  }
+
+  if (action === 'clear') {
+    await saveSynchronize({ delete: [], edit: [], add: [] });
+    log.success('Cleared all staged items from synchronize.json');
+    return;
+  }
+
+  if (action === 'review') {
+    const kept = { delete: [], edit: [], add: [] };
+
+    for (const category of ['delete', 'edit', 'add']) {
+      const items = sync[category] || [];
+      if (items.length === 0) continue;
+
+      for (const item of items) {
+        const label = item.Name || item.UID || item.metaPath || JSON.stringify(item).substring(0, 80);
+        const entity = item._entity || item.entity || '';
+        const desc = entity ? `[${category}] ${entity}: ${label}` : `[${category}] ${label}`;
+
+        const { keep } = await inquirer.prompt([{
+          type: 'confirm',
+          name: 'keep',
+          message: `Keep staged? ${desc}`,
+          default: true,
+        }]);
+
+        if (keep) {
+          kept[category].push(item);
+        } else {
+          log.dim(`  Discarded: ${desc}`);
+        }
+      }
+    }
+
+    await saveSynchronize(kept);
+    const remaining = kept.delete.length + kept.edit.length + kept.add.length;
+    if (remaining > 0) {
+      log.info(`${remaining} staged item(s) kept`);
+    } else {
+      log.success('All staged items discarded');
+    }
+    return;
+  }
+
+  // action === 'ignore' — do nothing, keep staged items
+  log.dim('  Keeping staged items');
+}
+
+/**
+ * Main clone workflow. Exported for use by init --clone.
+ */
+export async function performClone(source, options = {}) {
+  const config = await loadConfig();
+  const effectiveDomain = options.domain || config.domain;
+  let appJson;
+
+  // Step 1: Source mismatch detection
+  // Warn when the user provides an explicit source that differs from the stored one.
+  const storedCloneSource = await loadCloneSource();
+  if (source && storedCloneSource && source !== storedCloneSource) {
+    if (!options.force && !options.yes) {
+      log.warn('');
+      log.warn(`  ⚠  This project was previously cloned from: ${storedCloneSource}`);
+      log.warn(`     Requested source:                        ${source}`);
+      const inquirer = (await import('inquirer')).default;
+      const { confirmed } = await inquirer.prompt([{
+        type: 'confirm',
+        name: 'confirmed',
+        message: 'Clone from the new source anyway? This will update the stored clone source.',
+        default: false,
+      }]);
+      if (!confirmed) {
+        log.info('Clone aborted.');
+        return;
+      }
+    } else {
+      log.warn(`  ⚠  Clone source override: "${storedCloneSource}" → "${source}"`);
+    }
+  }
+
+  // Step 2: Load the app JSON — retry loop with fallback prompt on failure
+  let activeSource = source;
+  while (true) {
+    try {
+      appJson = await resolveAppSource(activeSource, options, config);
+      break;
+    } catch (err) {
+      if (options.yes || !process.stdin.isTTY) {
+        throw err;
+      }
+      log.warn('');
+      log.warn(`  ⚠  Source did not return expected results: ${err.message}`);
+      const inquirer = (await import('inquirer')).default;
+      const { fallback } = await inquirer.prompt([{
+        type: 'input',
+        name: 'fallback',
+        message: 'Enter another local file path or URL to retry (or leave empty to abort):',
+      }]);
+      if (!fallback || !fallback.trim()) {
+        throw new Error('Clone aborted: no valid source provided.');
+      }
+      activeSource = fallback.trim();
     }
   }
 
@@ -502,6 +696,9 @@ export async function performClone(source, options = {}) {
 
   log.success(`Cloning "${appJson.Name}" (${appJson.ShortName})`);
 
+  // Check for un-pushed staged items in synchronize.json
+  await checkPendingSynchronize(options);
+
   // Ensure sensitive files are gitignored
   await ensureGitignore(['.dbo/credentials.json', '.dbo/cookies.txt']);
 
@@ -512,6 +709,7 @@ export async function performClone(source, options = {}) {
     AppName: appJson.Name,
     AppShortName: appJson.ShortName,
   });
+  await saveCloneSource(activeSource || 'default');
   log.dim('  Updated .dbo/config.json with app metadata');
 
   // Detect and store ModifyKey for locked/production apps
@@ -637,7 +835,13 @@ export async function performClone(source, options = {}) {
     // Skip if entity filter is active and this entity doesn't match
     if (entityFilter && !entityFilter.has(entityName)) continue;
 
-    if (ENTITY_DIR_MAP[entityName]) {
+    if (entityName === 'extension') {
+      // Extension entities: descriptor-organized sub-directories
+      const refs = await processExtensionEntries(entries, structure, options, serverTz);
+      if (refs.length > 0) {
+        otherRefs[entityName] = refs;
+      }
+    } else if (ENTITY_DIR_MAP[entityName]) {
       // Entity types with project directories — process into their directory
       const refs = await processEntityDirEntries(entityName, entries, options, serverTz);
       if (refs.length > 0) {
@@ -756,16 +960,23 @@ async function resolvePlacementPreferences(appJson, options) {
     const answers = await inquirer.prompt(prompts);
     contentPlacement = contentPlacement || answers.contentPlacement || 'bin';
     mediaPlacement = mediaPlacement || answers.mediaPlacement || 'bin';
+  }
+
+  // Resolve defaults for any still-unset values
+  contentPlacement = contentPlacement || 'bin';
+  mediaPlacement = mediaPlacement || 'bin';
 
-    // Save to config for future clones
-    await saveClonePlacement({ mediaPlacement, contentPlacement });
-    log.dim('  Saved placement preferences to .dbo/config.json');
+  // Always persist resolved values — not just when prompts were shown.
+  // This ensures defaults are saved even when the app has no content/media yet,
+  // so subsequent clones that do have records skip the prompts.
+  if (!saved.contentPlacement || !saved.mediaPlacement) {
+    await saveClonePlacement({ contentPlacement, mediaPlacement });
+    if (prompts.length > 0) {
+      log.dim('  Saved placement preferences to .dbo/config.json');
+    }
   }
 
-  return {
-    contentPlacement: contentPlacement || 'bin',
-    mediaPlacement: mediaPlacement || 'bin',
-  };
+  return { contentPlacement, mediaPlacement };
 }
 
 /**
@@ -773,22 +984,45 @@ async function resolvePlacementPreferences(appJson, options) {
  */
 async function fetchAppFromServer(appShortName, options, config) {
   const client = new DboClient({ domain: options.domain, verbose: options.verbose });
-  log.info(`Fetching app "${appShortName}" from server...`);
 
-  const result = await client.get('/api/o/object_lookup_app__CALL', {
-    '_filter:AppShortName': appShortName,
-    '_format': 'json_raw',
-  });
+  const ora = (await import('ora')).default;
+  const spinner = ora(`Fetching app "${appShortName}" from server...`).start();
+
+  let result;
+  try {
+    result = await client.get(`/api/app/object/${appShortName}`);
+  } catch (err) {
+    spinner.fail(`Failed to fetch app "${appShortName}"`);
+    throw err;
+  }
 
   const data = result.payload || result.data;
-  const rows = Array.isArray(data) ? data : (data?.Rows || data?.rows || []);
 
-  if (rows.length === 0) {
+  // Handle all response shapes:
+  //   1. Array of rows: [{ UID, ShortName, ... }]
+  //   2. Object with Rows key: { Rows: [...] }
+  //   3. Single app object: { UID, ShortName, children, ... }
+  let appRecord;
+  if (Array.isArray(data)) {
+    appRecord = data.length > 0 ? data[0] : null;
+  } else if (data?.Rows?.length > 0) {
+    appRecord = data.Rows[0];
+  } else if (data?.rows?.length > 0) {
+    appRecord = data.rows[0];
+  } else if (data && typeof data === 'object' && (data.UID || data.ShortName)) {
+    // Single app object returned directly as payload
+    appRecord = data;
+  } else {
+    appRecord = null;
+  }
+
+  if (!appRecord) {
+    spinner.fail(`No app found with ShortName "${appShortName}"`);
     throw new Error(`No app found with ShortName "${appShortName}"`);
   }
 
-  log.success(`Found app on server`);
-  return rows[0];
+  spinner.succeed(`Found app on server`);
+  return appRecord;
 }
 
 /**
@@ -1082,11 +1316,16 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
       }
 
       if (bulkAction.value !== 'overwrite_all') {
+        const configWithTz = { ...config, ServerTimezone: serverTz };
         const localSyncTime = await getLocalSyncTime(metaPath);
-        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz);
+        const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
         if (serverNewer) {
-          const action = await promptChangeDetection(finalName, record, config);
+          const action = await promptChangeDetection(finalName, record, configWithTz, {
+            serverDate,
+            localDate: localSyncTime,
+          });
 
           if (action === 'skip') {
             log.dim(`  Skipped ${finalName}`);
@@ -1103,14 +1342,18 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
             bulkAction.value = 'overwrite_all';
           }
           if (action === 'compare') {
-            await inlineDiffAndMerge(record, metaPath, config);
+            await inlineDiffAndMerge(record, metaPath, configWithTz);
             refs.push({ uid: record.UID, metaPath });
             continue;
           }
         } else {
-          const locallyModified = await hasLocalModifications(metaPath, config);
+          const locallyModified = await hasLocalModifications(metaPath, configWithTz);
           if (locallyModified) {
-            const action = await promptChangeDetection(finalName, record, config, { localIsNewer: true });
+            const action = await promptChangeDetection(finalName, record, configWithTz, {
+              localIsNewer: true,
+              serverDate,
+              localDate: localSyncTime,
+            });
 
             if (action === 'skip') {
               log.dim(`  Kept local: ${finalName}`);
@@ -1127,7 +1370,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
               bulkAction.value = 'overwrite_all';
             }
             if (action === 'compare') {
-              await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+              await inlineDiffAndMerge(record, metaPath, configWithTz, { localIsNewer: true });
               refs.push({ uid: record.UID, metaPath });
               continue;
             }
@@ -1199,6 +1442,378 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
   return refs;
 }
 
+// ─── Extension Descriptor Sub-directory Processing ────────────────────────
+
+/**
+ * Scan extension records for descriptor_definition entries.
+ * Builds the mapping, persists to structure.json, always creates Extensions/Unsupported/,
+ * and creates sub-directories for every mapped descriptor.
+ * Returns the mapping object.
+ *
+ * @param {Object[]} extensionEntries
+ * @param {Object}   structure  - Current bin structure (from loadStructureFile)
+ * @returns {Promise<Object<string,string>>}
+ */
+async function buildDescriptorPrePass(extensionEntries, structure) {
+  const { mapping, warnings } = buildDescriptorMapping(extensionEntries);
+
+  for (const w of warnings) log.warn(`  descriptor_definition: ${w}`);
+
+  // Always create Unsupported/ — even if empty, users see at a glance what couldn't be mapped
+  await mkdir(EXTENSION_UNSUPPORTED_DIR, { recursive: true });
+  log.dim(`  ${EXTENSION_UNSUPPORTED_DIR}/`);
+
+  // Create one sub-directory per mapped descriptor name
+  for (const dirName of new Set(Object.values(mapping))) {
+    const fullDir = `${EXTENSION_DESCRIPTORS_DIR}/${dirName}`;
+    await mkdir(fullDir, { recursive: true });
+    log.dim(`  ${fullDir}/`);
+  }
+
+  await saveDescriptorMapping(structure, mapping);
+  log.dim(`  Saved descriptorMapping to .dbo/structure.json`);
+
+  return mapping;
+}
+
+/**
+ * Resolve filename column and content extraction preferences for one descriptor.
+ * Prompts the user on first use; saves to config.json; respects -y and --force.
+ *
+ * @param {string}   descriptor  - e.g. "documentation", "include", "control"
+ * @param {Object[]} records     - All extension records sharing this descriptor
+ * @param {Object}   options     - CLI options (options.yes, options.force)
+ * @returns {Promise<{ filenameCol: string, contentColsToExtract: Array<{col,ext}> }>}
+ */
+async function resolveDescriptorPreferences(descriptor, records, options) {
+  const sampleRecord = records[0];
+  const columns = Object.keys(sampleRecord)
+    .filter(k => k !== 'children' && !k.startsWith('_'));
+
+  // ── Filename column ──────────────────────────────────────────────────
+  let filenameCol;
+  const savedCol = await loadDescriptorFilenamePreference(descriptor);
+
+  if (options.yes) {
+    filenameCol = columns.includes('Name') ? 'Name'
+      : columns.includes('UID') ? 'UID' : columns[0];
+  } else if (savedCol && !options.force) {
+    filenameCol = savedCol;
+    log.dim(`  Filename column for "${descriptor}": "${filenameCol}" (saved)`);
+  } else {
+    const inquirer = (await import('inquirer')).default;
+    const defaultCol = columns.includes('Name') ? 'Name'
+      : columns.includes('UID') ? 'UID' : columns[0];
+    const { col } = await inquirer.prompt([{
+      type: 'list', name: 'col',
+      message: `Filename column for "${descriptor}" extensions:`,
+      choices: columns, default: defaultCol,
+    }]);
+    filenameCol = col;
+    await saveDescriptorFilenamePreference(descriptor, filenameCol);
+    log.dim(`  Saved filename column for "${descriptor}": "${filenameCol}"`);
+  }
+
+  // ── Content extraction ───────────────────────────────────────────────
+  const contentColsToExtract = [];
+  if (!options.yes) {
+    const base64Cols = [];
+    for (const record of records) {
+      for (const [key, value] of Object.entries(record)) {
+        if (key === 'children' || key.startsWith('_')) continue;
+        if (value && typeof value === 'object' && !Array.isArray(value)
+            && value.encoding === 'base64' && value.value !== null) {
+          if (!base64Cols.find(c => c.col === key)) {
+            let snippet = '';
+            try {
+              const decoded = Buffer.from(value.value, 'base64').toString('utf8');
+              snippet = decoded.substring(0, 80)
+                .replace(/[\x00-\x1f\x7f]/g, ' ')   // strip control chars (incl. \n, \r, \t, ESC)
+                .replace(/\s+/g, ' ')                  // collapse whitespace
+                .trim();
+              if (decoded.length > 80) snippet += '…';
+            } catch {}
+            base64Cols.push({ col: key, snippet });
+          }
+        }
+      }
+    }
+
+    if (base64Cols.length > 0) {
+      const savedExtractions = options.force
+        ? null
+        : await loadDescriptorContentExtractions(descriptor);
+      const newPreferences = savedExtractions ? { ...savedExtractions } : {};
+      let changed = false;
+      const inquirer = (await import('inquirer')).default;
+
+      for (const { col, snippet } of base64Cols) {
+        if (savedExtractions) {
+          const saved = savedExtractions[col];
+          if (saved === false) { log.dim(`  Skip "${col}" for "${descriptor}" (saved)`); continue; }
+          if (typeof saved === 'string') {
+            log.dim(`  Extracting "${col}" for "${descriptor}" as .${saved} (saved)`);
+            contentColsToExtract.push({ col, ext: saved });
+            continue;
+          }
+        }
+        const preview = snippet ? ` ("${snippet}")` : '';
+        const { extract } = await inquirer.prompt([{
+          type: 'confirm', name: 'extract',
+          message: `Extract column "${col}" (${descriptor}) as companion file?${preview}`,
+          default: true,
+        }]);
+        if (extract) {
+          const guessed = guessExtensionForDescriptor(descriptor, col);
+          const { ext } = await inquirer.prompt([{
+            type: 'input', name: 'ext',
+            message: `File extension for "${col}" (${descriptor}):`,
+            default: guessed,
+          }]);
+          const cleanExt = ext.replace(/^\./, '');
+          contentColsToExtract.push({ col, ext: cleanExt });
+          newPreferences[col] = cleanExt;
+        } else {
+          newPreferences[col] = false;
+        }
+        changed = true;
+      }
+      if (changed) await saveDescriptorContentExtractions(descriptor, newPreferences);
+    }
+  }
+
+  return { filenameCol, contentColsToExtract };
+}
+
+/**
+ * Guess a file extension based on descriptor name, with column-name fallback.
+ */
+function guessExtensionForDescriptor(descriptor, columnName) {
+  if (descriptor === 'documentation') return 'md';
+  if (descriptor === 'include') return 'html';
+  if (descriptor === 'control') return 'js';
+  return guessExtensionForColumn(columnName);
+}
+
+/**
+ * Resolve placement for documentation descriptor MD companion files.
+ * Prompts on first use; persists to config.json; respects -y and --force.
+ * @returns {Promise<'inline'|'root'>}
+ */
+async function resolveDocumentationPlacement(options) {
+  if (options.yes) return 'inline';
+
+  const saved = await loadExtensionDocumentationMDPlacement();
+  if (saved && !options.force) {
+    log.dim(`  Documentation MD placement: ${saved} (saved)`);
+    return saved;
+  }
+
+  const inquirer = (await import('inquirer')).default;
+  const { placement } = await inquirer.prompt([{
+    type: 'list', name: 'placement',
+    message: 'Where should extracted documentation MD files be placed?',
+    choices: [
+      { name: '/Documentation/<filename>.md  — project root (recommended)', value: 'root' },
+      { name: 'Extensions/Documentation/<filename>.md  — inline alongside metadata', value: 'inline' },
+    ],
+    default: 'root',
+  }]);
+
+  await saveExtensionDocumentationMDPlacement(placement);
+  log.dim(`  Saved documentation MD placement: "${placement}"`);
+  return placement;
+}
+
+/**
+ * Process extension entity records into descriptor-organized sub-directories.
+ *
+ * @param {Object[]} entries   - All extension records from appJson.children.extension
+ * @param {Object}   structure - Bin structure from loadStructureFile()
+ * @param {Object}   options   - CLI options (options.yes, options.force, options.entity)
+ * @param {string}   serverTz  - Server timezone for timestamp syncing
+ * @returns {Promise<Array<{uid, metaPath}>>}
+ */
+async function processExtensionEntries(entries, structure, options, serverTz) {
+  if (!entries || entries.length === 0) return [];
+
+  // --descriptor-types false → skip descriptor sorting, use flat Extensions/ via generic path
+  if (options.descriptorTypes === 'false') {
+    log.info(`Processing ${entries.length} extension record(s) (flat mode)...`);
+    return await processEntityDirEntries('extension', entries, options, serverTz);
+  }
+
+  log.info(`Processing ${entries.length} extension record(s)...`);
+
+  // Step A: Pre-pass — build mapping + create directories
+  const mapping = await buildDescriptorPrePass(entries, structure);
+
+  // Clear documentation preferences when --force is used with --documentation-only
+  if (options.documentationOnly && options.force) {
+    await saveDescriptorFilenamePreference('documentation', null);
+    await saveDescriptorContentExtractions('documentation', null);
+    await saveExtensionDocumentationMDPlacement(null);
+    log.info('  --force: cleared saved documentation preferences');
+  }
+
+  // Step B: Group records by descriptor
+  const groups = new Map(); // descriptor → { dir, records[] }
+  for (const record of entries) {
+    const descriptor = record.Descriptor || '__unsupported__';
+
+    // --documentation-only: skip non-documentation records
+    if (options.documentationOnly && descriptor !== 'documentation') continue;
+
+    if (!groups.has(descriptor)) {
+      groups.set(descriptor, { dir: resolveExtensionSubDir(record, mapping), records: [] });
+    }
+    groups.get(descriptor).records.push(record);
+  }
+
+  // Step C: Resolve preferences per descriptor (prompts fire here, before file writes)
+  const descriptorPrefs = new Map();
+  let docPlacement = 'inline';
+
+  for (const [descriptor, { records }] of groups.entries()) {
+    if (descriptor === '__unsupported__') {
+      descriptorPrefs.set(descriptor, { filenameCol: null, contentColsToExtract: [] });
+      continue;
+    }
+    const prefs = await resolveDescriptorPreferences(descriptor, records, options);
+    descriptorPrefs.set(descriptor, prefs);
+
+    if (descriptor === 'documentation') {
+      docPlacement = await resolveDocumentationPlacement(options);
+      if (docPlacement === 'root') {
+        await mkdir(DOCUMENTATION_DIR, { recursive: true });
+        log.dim(`  ${DOCUMENTATION_DIR}/`);
+      }
+    }
+  }
+
+  // Step D: Write files, one group at a time
+  const refs = [];
+  const bulkAction = { value: null };
+  const config = await loadConfig();
+
+  for (const [descriptor, { dir, records }] of groups.entries()) {
+    const { filenameCol, contentColsToExtract } = descriptorPrefs.get(descriptor);
+    const useRootDoc = (descriptor === 'documentation' && docPlacement === 'root');
+    const mdColInfo = useRootDoc ? contentColsToExtract.find(c => c.ext === 'md') : null;
+
+    log.info(`Processing ${records.length} "${descriptor}" extension(s) → ${dir}/`);
+
+    for (const record of records) {
+      // Resolve filename
+      let name;
+      if (filenameCol && record[filenameCol] != null) {
+        name = sanitizeFilename(String(record[filenameCol]));
+      } else if (record.Name) {
+        name = sanitizeFilename(String(record.Name));
+      } else {
+        name = sanitizeFilename(String(record.UID || 'untitled'));
+      }
+
+      const uid = record.UID || 'untitled';
+      const finalName = name === uid ? uid : `${name}.${uid}`;
+      const metaPath = join(dir, `${finalName}.metadata.json`);
+
+      // Change detection — same pattern as processEntityDirEntries()
+      const hasNewExtractions = contentColsToExtract.length > 0;
+      if (await fileExists(metaPath) && !options.yes && !hasNewExtractions) {
+        if (bulkAction.value === 'skip_all') {
+          log.dim(`  Skipped ${finalName}`);
+          refs.push({ uid: record.UID, metaPath });
+          continue;
+        }
+        if (bulkAction.value !== 'overwrite_all') {
+          const cfgWithTz = { ...config, ServerTimezone: serverTz };
+          const localSyncTime = await getLocalSyncTime(metaPath);
+          const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, cfgWithTz);
+          const serverDate = parseServerDate(record._LastUpdated, serverTz);
+
+          if (serverNewer) {
+            const action = await promptChangeDetection(finalName, record, cfgWithTz, { serverDate, localDate: localSyncTime });
+            if (action === 'skip') { refs.push({ uid: record.UID, metaPath }); continue; }
+            if (action === 'skip_all') { bulkAction.value = 'skip_all'; refs.push({ uid: record.UID, metaPath }); continue; }
+            if (action === 'overwrite_all') { bulkAction.value = 'overwrite_all'; }
+            if (action === 'compare') { await inlineDiffAndMerge(record, metaPath, cfgWithTz); refs.push({ uid: record.UID, metaPath }); continue; }
+          } else {
+            const localModified = await hasLocalModifications(metaPath, cfgWithTz);
+            if (localModified) {
+              const action = await promptChangeDetection(finalName, record, cfgWithTz, { localIsNewer: true, serverDate, localDate: localSyncTime });
+              if (action === 'skip') { refs.push({ uid: record.UID, metaPath }); continue; }
+              if (action === 'skip_all') { bulkAction.value = 'skip_all'; refs.push({ uid: record.UID, metaPath }); continue; }
+              if (action === 'overwrite_all') { bulkAction.value = 'overwrite_all'; }
+              if (action === 'compare') { await inlineDiffAndMerge(record, metaPath, cfgWithTz, { localIsNewer: true }); refs.push({ uid: record.UID, metaPath }); continue; }
+            } else {
+              log.dim(`  Up to date: ${finalName}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+          }
+        }
+      }
+
+      // Build metadata object
+      const meta = {};
+      const extractedCols = [];
+
+      for (const [key, value] of Object.entries(record)) {
+        if (key === 'children') continue;
+
+        const extractInfo = contentColsToExtract.find(c => c.col === key);
+        if (extractInfo && value && typeof value === 'object'
+            && value.encoding === 'base64' && value.value !== null) {
+          const decoded = resolveContentValue(value);
+          if (decoded) {
+            let colFilePath, refValue;
+
+            if (mdColInfo && extractInfo.col === mdColInfo.col) {
+              // Root placement: Documentation/<name>.md
+              const docFileName = `${name}.md`;
+              colFilePath = join(DOCUMENTATION_DIR, docFileName);
+              refValue = `@/${DOCUMENTATION_DIR}/${docFileName}`;
+            } else {
+              const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
+              colFilePath = join(dir, colFileName);
+              refValue = `@${colFileName}`;
+            }
+
+            meta[key] = refValue;
+            await writeFile(colFilePath, decoded);
+            extractedCols.push(key);
+            if (serverTz) {
+              try { await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz); } catch {}
+            }
+            log.dim(`  → ${colFilePath}`);
+            continue;
+          }
+        }
+
+        // Inline or non-extraction columns
+        if (value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64') {
+          meta[key] = resolveContentValue(value);
+        } else {
+          meta[key] = value;
+        }
+      }
+
+      meta._entity = 'extension';
+      if (extractedCols.length > 0) meta._contentColumns = extractedCols;
+
+      await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+      if (serverTz) {
+        try { await setFileTimestamps(metaPath, record._CreatedOn, record._LastUpdated, serverTz); } catch {}
+      }
+      log.success(`Saved ${metaPath}`);
+      refs.push({ uid: record.UID, metaPath });
+    }
+  }
+
+  return refs;
+}
+
 /**
  * Process media entries: download binary files from server + create metadata.
  * Media uses Filename (not Name) and files are fetched via /api/media/{uid}.
@@ -1349,11 +1964,20 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
       }
 
       if (mediaBulkAction.value !== 'overwrite_all') {
+        // Use explicit serverTz (not config.ServerTimezone) to ensure consistency
+        // with how setFileTimestamps set the mtime — the two must use the same timezone.
+        const configWithTz = { ...config, ServerTimezone: serverTz };
         const localSyncTime = await getLocalSyncTime(metaPath);
-        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz);
+        const serverDate = parseServerDate(record._LastUpdated, serverTz);
+        const diffable = isDiffable(ext);
 
         if (serverNewer) {
-          const action = await promptChangeDetection(dedupName, record, config);
+          const action = await promptChangeDetection(dedupName, record, configWithTz, {
+            diffable,
+            serverDate,
+            localDate: localSyncTime,
+          });
 
           if (action === 'skip') {
             log.dim(`  Skipped ${finalFilename}`);
@@ -1371,15 +1995,21 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
           }
           if (action === 'compare') {
             // For binary media, show metadata diffs only
-            await inlineDiffAndMerge(record, metaPath, config);
+            await inlineDiffAndMerge(record, metaPath, configWithTz);
             refs.push({ uid: record.UID, metaPath });
             continue;
           }
         } else {
           // Server _LastUpdated hasn't changed — check for local modifications
-          const locallyModified = await hasLocalModifications(metaPath, config);
+          const locallyModified = await hasLocalModifications(metaPath, configWithTz);
           if (locallyModified) {
-            const action = await promptChangeDetection(dedupName, record, config, { localIsNewer: true });
+            const localDate = localSyncTime; // mtime already fetched above
+            const action = await promptChangeDetection(dedupName, record, configWithTz, {
+              localIsNewer: true,
+              diffable,
+              serverDate,
+              localDate,
+            });
 
             if (action === 'skip') {
               log.dim(`  Kept local: ${finalFilename}`);
@@ -1396,7 +2026,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
               mediaBulkAction.value = 'overwrite_all';
             }
             if (action === 'compare') {
-              await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+              await inlineDiffAndMerge(record, metaPath, configWithTz, { localIsNewer: true });
               refs.push({ uid: record.UID, metaPath });
               continue;
             }
@@ -1454,10 +2084,13 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
     log.dim(`  → ${metaPath}`);
 
-    // Set file timestamps from server dates
+    // Set file timestamps from server dates (independent try-catch so one failure
+    // doesn't prevent the other — metaPath mtime is the sync baseline for comparisons)
     if (serverTz && (record._CreatedOn || record._LastUpdated)) {
       try {
         await setFileTimestamps(filePath, record._CreatedOn, record._LastUpdated, serverTz);
+      } catch { /* non-critical */ }
+      try {
         await setFileTimestamps(metaPath, record._CreatedOn, record._LastUpdated, serverTz);
       } catch { /* non-critical */ }
     }
@@ -1531,6 +2164,41 @@ async function processRecord(entityName, record, structure, options, usedNames,
   }
   // If still no extension, ext remains '' (no extension)
 
+  // If no extension determined and Content column has data, prompt user to choose one
+  if (!ext && !options.yes && record.Content) {
+    const cv = record.Content;
+    const hasContentData = cv && (
+      (typeof cv === 'object' && cv.value !== null && cv.value !== undefined) ||
+      (typeof cv === 'string' && cv.length > 0)
+    );
+    if (hasContentData) {
+      // Decode a snippet for preview
+      let snippet = '';
+      try {
+        const decoded = resolveContentValue(cv);
+        if (decoded) {
+          snippet = decoded.substring(0, 80).replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
+          if (decoded.length > 80) snippet += '...';
+        }
+      } catch { /* ignore decode errors */ }
+      const preview = snippet ? ` (${snippet})` : '';
+      const VALID_CONTENT_EXTENSIONS = ['css', 'js', 'html', 'xml', 'txt', 'md', 'cs', 'json', 'sql'];
+      const inquirer = (await import('inquirer')).default;
+      const { chosenExt } = await inquirer.prompt([{
+        type: 'list',
+        name: 'chosenExt',
+        message: `No extension found for "${record.Name || record.UID}". Choose a file extension for the Content:${preview}`,
+        choices: [
+          ...VALID_CONTENT_EXTENSIONS.map(e => ({ name: `.${e}`, value: e })),
+          { name: 'No extension (skip)', value: '' },
+        ],
+      }]);
+      if (chosenExt) {
+        ext = chosenExt;
+      }
+    }
+  }
+
   // Avoid double extension: if name already ends with .ext, strip it
   if (ext) {
     const extWithDot = `.${ext}`;
@@ -1630,12 +2298,17 @@ async function processRecord(entityName, record, structure, options, usedNames,
     }
 
     if (bulkAction.value !== 'overwrite_all') {
-      const localSyncTime = await getLocalSyncTime(metaPath);
       const config = await loadConfig();
-      const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+      const configWithTz = { ...config, ServerTimezone: serverTz };
+      const localSyncTime = await getLocalSyncTime(metaPath);
+      const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz);
+      const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
       if (serverNewer) {
-        const action = await promptChangeDetection(finalName, record, config);
+        const action = await promptChangeDetection(finalName, record, configWithTz, {
+          serverDate,
+          localDate: localSyncTime,
+        });
 
         if (action === 'skip') {
           log.dim(`  Skipped ${finalName}.${ext}`);
@@ -1651,16 +2324,20 @@ async function processRecord(entityName, record, structure, options, usedNames,
           // Fall through to write
         }
         if (action === 'compare') {
-          await inlineDiffAndMerge(record, metaPath, config);
+          await inlineDiffAndMerge(record, metaPath, configWithTz);
           return { uid: record.UID, metaPath };
         }
         // 'overwrite' falls through to normal write
       } else {
         // Server _LastUpdated hasn't changed since last sync.
         // Check if local content files were modified (user edits).
-        const locallyModified = await hasLocalModifications(metaPath, config);
+        const locallyModified = await hasLocalModifications(metaPath, configWithTz);
         if (locallyModified) {
-          const action = await promptChangeDetection(finalName, record, config, { localIsNewer: true });
+          const action = await promptChangeDetection(finalName, record, configWithTz, {
+            localIsNewer: true,
+            serverDate,
+            localDate: localSyncTime,
+          });
 
           if (action === 'skip') {
             log.dim(`  Kept local: ${finalName}.${ext}`);
@@ -1675,7 +2352,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
             bulkAction.value = 'overwrite_all';
           }
           if (action === 'compare') {
-            await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+            await inlineDiffAndMerge(record, metaPath, configWithTz, { localIsNewer: true });
             return { uid: record.UID, metaPath };
           }
           // 'overwrite' falls through to normal write
@@ -1728,6 +2405,13 @@ async function processRecord(entityName, record, structure, options, usedNames,
     meta._contentColumns = ['Content'];
   }
 
+  // If the extension picker chose an extension (record.Extension was null),
+  // set it in metadata only — not in the record — so the baseline preserves
+  // the server's null and push detects the change.
+  if (ext && !record.Extension) {
+    meta.Extension = ext;
+  }
+
   await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
   log.dim(`  → ${metaPath}`);
 
@@ -2073,6 +2757,7 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
   const refs = [];
   const bulkAction = { value: null };
   const config = await loadConfig();
+  const configWithTz = { ...config, ServerTimezone: serverTz };
   // When --force flag is set, skip change detection and re-process all files
   const forceReprocess = !!options.force;
 
@@ -2125,21 +2810,22 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
       }
       if (bulkAction.value !== 'overwrite_all') {
         const localSyncTime = await getLocalSyncTime(rootMetaPath);
-        const serverNewer = isServerNewer(localSyncTime, output._LastUpdated, config);
+        const serverNewer = isServerNewer(localSyncTime, output._LastUpdated, configWithTz);
+        const serverDate = parseServerDate(output._LastUpdated, serverTz);
         if (serverNewer) {
-          const action = await promptChangeDetection(rootBasename, output, config);
+          const action = await promptChangeDetection(rootBasename, output, configWithTz, { serverDate, localDate: localSyncTime });
           if (action === 'skip') { refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
           if (action === 'skip_all') { bulkAction.value = 'skip_all'; refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
           if (action === 'overwrite_all') { bulkAction.value = 'overwrite_all'; }
-          if (action === 'compare') { await inlineDiffAndMerge(output, rootMetaPath, config); refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
+          if (action === 'compare') { await inlineDiffAndMerge(output, rootMetaPath, configWithTz); refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
         } else {
-          const locallyModified = await hasLocalModifications(rootMetaPath, config);
+          const locallyModified = await hasLocalModifications(rootMetaPath, configWithTz);
           if (locallyModified) {
-            const action = await promptChangeDetection(rootBasename, output, config, { localIsNewer: true });
+            const action = await promptChangeDetection(rootBasename, output, configWithTz, { localIsNewer: true, serverDate, localDate: localSyncTime });
             if (action === 'skip') { refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
             if (action === 'skip_all') { bulkAction.value = 'skip_all'; refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
             if (action === 'overwrite_all') { bulkAction.value = 'overwrite_all'; }
-            if (action === 'compare') { await inlineDiffAndMerge(output, rootMetaPath, config, { localIsNewer: true }); refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
+            if (action === 'compare') { await inlineDiffAndMerge(output, rootMetaPath, configWithTz, { localIsNewer: true }); refs.push({ uid: output.UID, metaPath: rootMetaPath }); continue; }
           } else {
             log.dim(`  Up to date: ${rootBasename}`);
             refs.push({ uid: output.UID, metaPath: rootMetaPath });