@dboio/cli 0.15.3 → 0.17.0

package/src/commands/clone.js

@@ -1,10 +1,10 @@
 import { Command } from 'commander';
-import { readFile, writeFile, appendFile, mkdir, access, readdir, rename, stat } from 'fs/promises';
+import { readFile, writeFile, appendFile, mkdir, access, readdir, rename, stat, utimes } from 'fs/promises';
 import { join, basename, extname, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import { DboClient } from '../lib/client.js';
 import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference, saveCloneSource, loadCloneSource, saveDescriptorFilenamePreference, loadDescriptorFilenamePreference, saveDescriptorContentExtractions, loadDescriptorContentExtractions, saveExtensionDocumentationMDPlacement, loadExtensionDocumentationMDPlacement } from '../lib/config.js';
-import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, SCAFFOLD_DIRS, ENTITY_DIR_NAMES, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir, resolveEntityDirPath } from '../lib/structure.js';
+import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, SCAFFOLD_DIRS, ENTITY_DIR_NAMES, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir, resolveEntityDirPath, resolveFieldValue } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
 import { buildUidFilename, buildContentFileName, buildMetaFilename, isMetadataFile, parseMetaFilename, stripUidFromFilename, hasUidInFilename } from '../lib/filenames.js';
 import { setFileTimestamps, parseServerDate } from '../lib/timestamps.js';
@@ -12,9 +12,12 @@ import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDet
 import { loadIgnore } from '../lib/ignore.js';
 import { checkDomainChange } from '../lib/domain-guard.js';
 import { applyTrashIcon, ensureTrashIcon, tagProjectFiles } from '../lib/tagging.js';
-import { loadMetadataTemplates, saveMetadataTemplates, getTemplateCols, setTemplateCols, buildTemplateFromCloneRecord } from '../lib/metadata-templates.js';
+import { loadMetadataSchema, saveMetadataSchema, getTemplateCols, setTemplateCols, buildTemplateFromCloneRecord, generateMetadataFromSchema, parseReferenceExpression, mergeDescriptorSchemaFromDependencies } from '../lib/metadata-schema.js';
+import { fetchSchema, loadSchema, saveSchema, isSchemaStale, SCHEMA_FILE } from '../lib/schema.js';
 import { runPendingMigrations } from '../lib/migrations.js';
 import { upsertDeployEntry } from '../lib/deploy-config.js';
+import { syncDependencies, parseDependenciesColumn } from '../lib/dependencies.js';
+import { mergeDependencies } from '../lib/config.js';
 
 /**
  * Resolve a column value that may be base64-encoded.
@@ -124,7 +127,7 @@ export async function detectAndTrashOrphans(appJson, ig, sync, options) {
     const metaDir = dirname(metaPath);
     const filesToMove = [metaPath];
 
-    for (const col of (meta._contentColumns || [])) {
+    for (const col of (meta._companionReferenceColumns || meta._contentColumns || [])) {
       const ref = meta[col];
       if (ref && String(ref).startsWith('@')) {
         const refName = String(ref).substring(1);
@@ -411,7 +414,7 @@ async function detectAndRenameLegacyCompanions(metaPath, meta) {
   if (!uid) return false;
 
   const metaDir = dirname(metaPath);
-  const contentCols = [...(meta._contentColumns || [])];
+  const contentCols = [...(meta._companionReferenceColumns || meta._contentColumns || [])];
   if (meta._mediaFile) contentCols.push('_mediaFile');
   let metaChanged = false;
 
@@ -475,10 +478,13 @@ async function detectAndRenameLegacyCompanions(metaPath, meta) {
     }
   }
 
-  // Rewrite metadata file if @references were updated
+  // Rewrite metadata file if @references were updated (preserve timestamps
+  // so the write doesn't cause false "local changes" during change detection)
   if (metaChanged) {
     try {
+      const before = await stat(metaPath);
       await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+      await utimes(metaPath, before.atime, before.mtime);
     } catch { /* non-critical */ }
   }
 
@@ -519,7 +525,7 @@ async function trashOrphanedLegacyCompanions() {
         // Read metadata and collect all @references
         try {
           const meta = JSON.parse(await readFile(full, 'utf8'));
-          const cols = [...(meta._contentColumns || [])];
+          const cols = [...(meta._companionReferenceColumns || meta._contentColumns || [])];
           if (meta._mediaFile) cols.push('_mediaFile');
           for (const col of cols) {
             const ref = meta[col];
@@ -804,7 +810,11 @@ export const cloneCommand = new Command('clone')
   .option('-y, --yes', 'Auto-accept all prompts')
   .option('--media-placement <placement>', 'Set media placement: fullpath or binpath (default: bin)')
   .option('-v, --verbose', 'Show HTTP request details')
+  .option('--schema', 'Re-fetch schema.json from server before cloning')
   .option('--no-migrate', 'Skip pending migrations for this invocation')
+  .option('--no-deps', 'Skip dependency cloning after clone')
+  .option('--dependencies <apps>', 'Sync specific dependency apps (comma-separated short-names)')
+  .option('--configure', 'Re-prompt for filename columns and companion file extraction preferences')
   .action(async (source, options) => {
     try {
       await runPendingMigrations(options);
@@ -1009,6 +1019,35 @@ export async function performClone(source, options = {}) {
   const effectiveDomain = options.domain || config.domain;
   let appJson;
 
+  // Fetch schema if missing, explicitly requested, or server has a newer version
+  let schema = await loadSchema();
+  let shouldFetch = !schema || options.schema;
+  if (!shouldFetch && schema) {
+    try {
+      shouldFetch = await isSchemaStale({ domain: effectiveDomain, verbose: options.verbose });
+      if (shouldFetch) log.dim(`  Server schema is newer — refreshing schema.json`);
+    } catch {
+      // Can't check — continue with local schema
+    }
+  }
+  if (shouldFetch) {
+    try {
+      schema = await fetchSchema({ domain: effectiveDomain, verbose: options.verbose });
+      await saveSchema(schema);
+      log.dim(`  Saved schema.json`);
+    } catch (err) {
+      if (!schema) log.warn(`  Could not fetch schema: ${err.message}`);
+      // Continue with stale schema or null
+    }
+  }
+
+  // Regenerate metadata_schema.json for any new entity types
+  if (schema) {
+    const existing = await loadMetadataSchema();
+    const updated = generateMetadataFromSchema(schema, existing ?? {});
+    await saveMetadataSchema(updated);
+  }
+
   // Step 1: Source mismatch detection (skip in pull mode)
   // Warn when the user provides an explicit source that differs from the stored one.
   const storedCloneSource = options.pullMode ? null : await loadCloneSource();
@@ -1092,6 +1131,16 @@ export async function performClone(source, options = {}) {
     });
     await saveCloneSource(activeSource || 'default');
     log.dim('  Updated .dbo/config.json with app metadata');
+
+    // Merge Dependencies from app.json into .dbo/config.json
+    // Always ensure at least ["_system"] is persisted
+    const fromApp = parseDependenciesColumn(appJson.Dependencies);
+    if (fromApp.length > 0) {
+      await mergeDependencies(fromApp);
+      log.dim(`  Merged app.json Dependencies: ${fromApp.join(', ')}`);
+    } else {
+      await mergeDependencies([]);
+    }
   }
 
   // Detect and store ModifyKey for locked/production apps (skip in pull mode)
@@ -1120,6 +1169,41 @@ export async function performClone(source, options = {}) {
     await updatePackageJson(appJson, config);
   }
 
+  // Step 3b: Sync dependency apps — before main content processing (skip in pull mode, entity filter, no-deps)
+  if (!options.pullMode && !options.entity && !options.noDeps) {
+    const explicitDeps = options.dependencies
+      ? options.dependencies.split(',').map(s => s.trim().toLowerCase()).filter(Boolean)
+      : null;
+    if (explicitDeps && explicitDeps.length > 0) {
+      await mergeDependencies(explicitDeps);
+    }
+    try {
+      await syncDependencies({
+        domain: effectiveDomain,
+        force: explicitDeps ? true : options.force,
+        schema: options.schema,
+        verbose: options.verbose,
+        systemSchemaPath: join(process.cwd(), SCHEMA_FILE),
+        only: explicitDeps || undefined,
+      });
+    } catch (err) {
+      log.warn(`  Dependency sync failed: ${err.message}`);
+    }
+
+    // Merge extension descriptor definitions from dependency schemas
+    // (e.g. operator provides descriptor_definition entries for apps that lack their own)
+    try {
+      const localSchema = await loadMetadataSchema();
+      if (localSchema) {
+        const merged = await mergeDescriptorSchemaFromDependencies(localSchema);
+        if (merged) {
+          await saveMetadataSchema(localSchema);
+          log.dim('  Merged extension descriptor schema from dependencies');
+        }
+      }
+    } catch { /* non-critical */ }
+  }
+
   // Step 4: Create default project directories + bin structure
   const bins = appJson.children.bin || [];
   const structure = buildBinHierarchy(bins, appJson.AppID);
@@ -1578,26 +1662,23 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
   const usedNames = new Map(); // name → count, for collision resolution
   const config = await loadConfig();
 
-  // Determine filename column: saved preference, or prompt, or default
+  // Determine filename column: saved preference, or auto-default, or prompt (--configure)
   let filenameCol = null;
 
   if (options.yes) {
     // Non-interactive: use Name, fallback UID
     filenameCol = null; // will resolve per-record below
   } else {
-    // Check saved preference
-    const savedCol = await loadEntityDirPreference(entityName);
+    // Check saved preference (skip if --configure to allow re-prompting)
+    const savedCol = options.configure ? null : await loadEntityDirPreference(entityName);
     if (savedCol) {
       filenameCol = savedCol;
       log.dim(`  Using saved filename column "${filenameCol}" for ${entityName}`);
     } else {
-      // Prompt user to pick a filename column from available columns
       const sampleRecord = entries[0];
       const columns = Object.keys(sampleRecord).filter(k => k !== 'children' && !k.startsWith('_'));
 
       if (columns.length > 0) {
-        const inquirer = (await import('inquirer')).default;
-
         // Find best default (app_version prioritizes Number → Name → UID)
         const defaultCol = (entityName === 'app_version' && columns.includes('Number')) ? 'Number'
           : columns.includes('Name') ? 'Name'
@@ -1605,18 +1686,25 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
           : columns.includes('UID') ? 'UID'
           : columns[0];
 
-        const { col } = await inquirer.prompt([{
-          type: 'list',
-          name: 'col',
-          message: `Which column should be used as the filename for ${entityName} records?`,
-          choices: columns,
-          default: defaultCol,
-        }]);
-        filenameCol = col;
+        if (options.configure) {
+          // --configure: prompt user to pick
+          const inquirer = (await import('inquirer')).default;
+          const { col } = await inquirer.prompt([{
+            type: 'list',
+            name: 'col',
+            message: `Which column should be used as the filename for ${entityName} records?`,
+            choices: columns,
+            default: defaultCol,
+          }]);
+          filenameCol = col;
+        } else {
+          // Auto-apply best default silently
+          filenameCol = defaultCol;
+        }
 
         // Save preference for future clones
         await saveEntityDirPreference(entityName, filenameCol);
-        log.dim(`  Saved filename column preference for ${entityName}`);
+        log.dim(`  Using filename column "${filenameCol}" for ${entityName}`);
       }
     }
   }
@@ -1646,13 +1734,11 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
     }
 
     if (base64Cols.length > 0) {
-      // Load saved content extraction preferences
-      const savedExtractions = await loadEntityContentExtractions(entityName);
+      // Load saved content extraction preferences (skip if --configure to allow re-prompting)
+      const savedExtractions = options.configure ? null : await loadEntityContentExtractions(entityName);
       const newPreferences = savedExtractions ? { ...savedExtractions } : {};
       let hasNewChoices = false;
 
-      const inquirer = (await import('inquirer')).default;
-
       // Prompt per column: show snippet and ask extract yes/no + extension
       for (const { col, snippet } of base64Cols) {
         // Check if we have a saved preference for this column
@@ -1670,32 +1756,42 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
           }
         }
 
-        // No saved preference - prompt the user
-        const preview = snippet ? ` (${snippet})` : '';
+        // No saved preference
         const guessed = guessExtensionForColumn(col);
 
-        const { extract } = await inquirer.prompt([{
-          type: 'confirm',
-          name: 'extract',
-          message: `Extract "${col}" as companion file?${preview}`,
-          default: true,
-        }]);
+        if (options.configure) {
+          // --configure: prompt user
+          const preview = snippet ? ` (${snippet})` : '';
+          const inquirer = (await import('inquirer')).default;
 
-        if (extract) {
-          const { ext } = await inquirer.prompt([{
-            type: 'input',
-            name: 'ext',
-            message: `File extension for "${col}":`,
-            default: guessed,
+          const { extract } = await inquirer.prompt([{
+            type: 'confirm',
+            name: 'extract',
+            message: `Extract "${col}" as companion file?${preview}`,
+            default: true,
           }]);
-          const cleanExt = ext.replace(/^\./, '');
-          contentColsToExtract.push({ col, ext: cleanExt });
-          newPreferences[col] = cleanExt;
-          hasNewChoices = true;
+
+          if (extract) {
+            const { ext } = await inquirer.prompt([{
+              type: 'input',
+              name: 'ext',
+              message: `File extension for "${col}":`,
+              default: guessed,
+            }]);
+            const cleanExt = ext.replace(/^\./, '');
+            contentColsToExtract.push({ col, ext: cleanExt });
+            newPreferences[col] = cleanExt;
+            hasNewChoices = true;
+          } else {
+            newPreferences[col] = false;
+            hasNewChoices = true;
+          }
         } else {
-          // User chose not to extract - save this preference too
-          newPreferences[col] = false;
+          // Auto-apply: extract with guessed extension
+          contentColsToExtract.push({ col, ext: guessed });
+          newPreferences[col] = guessed;
           hasNewChoices = true;
+          log.dim(`  Auto-extracting "${col}" as .${guessed} file`);
         }
       }
 
@@ -1788,7 +1884,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
         const configWithTz = { ...config, ServerTimezone: serverTz };
         const localSyncTime = await getLocalSyncTime(metaPath);
 
-        // If local metadata has no _LastUpdated (e.g. from dbo add), treat as server-newer
+        // If local metadata has no _LastUpdated (e.g. from dbo adopt), treat as server-newer
         let localMissingLastUpdated = false;
         try {
           const localMeta = JSON.parse(await readFile(metaPath, 'utf8'));
@@ -1799,7 +1895,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
         const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
         if (serverNewer) {
-          // Incomplete metadata (no _LastUpdated) from dbo add — auto-accept without prompting
+          // Incomplete metadata (no _LastUpdated) from dbo adopt — auto-accept without prompting
           if (localMissingLastUpdated) {
             log.dim(`  Completing metadata: ${name}`);
             // Fall through to write
@@ -1906,7 +2002,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
 
     meta._entity = entityName;
     if (extractedContentCols.length > 0) {
-      meta._contentColumns = extractedContentCols;
+      meta._companionReferenceColumns = extractedContentCols;
     }
 
     await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
@@ -1924,7 +2020,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
 
   // --- Seed metadata template if not already present ---
   if (entries.length > 0) {
-    const templates = await loadMetadataTemplates();
+    const templates = await loadMetadataSchema();
     if (templates !== null) {
       const existing = getTemplateCols(templates, entityName, null);
       if (!existing) {
@@ -1932,7 +2028,7 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
         const extractedColNames = contentColsToExtract.map(c => c.col);
         const cols = buildTemplateFromCloneRecord(firstRecord, extractedColNames);
         setTemplateCols(templates, entityName, null, cols);
-        await saveMetadataTemplates(templates);
+        await saveMetadataSchema(templates);
       }
     }
   }
@@ -1942,25 +2038,51 @@ async function processEntityDirEntries(entityName, entries, options, serverTz) {
 
 // ─── Extension Descriptor Sub-directory Processing ────────────────────────
 
+/**
+ * Parse form-control-code and form-control-title from a descriptor_definition String5 value.
+ * Returns { colToExt: Map<col,ext>, colToTitle: Map<col,title> }.
+ */
+function parseFormControlCode(string5) {
+  if (!string5) return { colToExt: new Map(), colToTitle: new Map() };
+  const colToExt = new Map();
+  const colToTitle = new Map();
+  try {
+    const params = new URLSearchParams(string5.startsWith('&') ? string5.slice(1) : string5);
+    const codeStr = params.get('form-control-code');
+    if (codeStr) {
+      for (const pair of codeStr.split(',')) {
+        const [col, ext] = pair.split('|');
+        if (col?.trim() && ext?.trim()) colToExt.set(col.trim(), ext.trim().toLowerCase());
+      }
+    }
+    const titleStr = params.get('form-control-title');
+    if (titleStr) {
+      for (const pair of titleStr.split(',')) {
+        const [col, title] = pair.split('|');
+        if (col?.trim() && title?.trim()) colToTitle.set(col.trim(), title.trim());
+      }
+    }
+  } catch { /* malformed params — ignore */ }
+  return { colToExt, colToTitle };
+}
+
 /**
  * Scan extension records for descriptor_definition entries.
- * Builds the mapping, persists to structure.json, always creates Extensions/Unsupported/,
- * and creates sub-directories for every mapped descriptor.
+ * Builds the mapping, persists to structure.json, creates sub-directories
+ * for every mapped descriptor, and populates metadata_schema.json from
+ * form-control-code entries.
  * Returns the mapping object.
  *
  * @param {Object[]} extensionEntries
- * @param {Object}   structure  - Current bin structure (from loadStructureFile)
+ * @param {Object}   structure      - Current bin structure (from loadStructureFile)
+ * @param {Object}   metadataSchema - Current metadata schema (from loadMetadataSchema)
  * @returns {Promise<Object<string,string>>}
  */
-async function buildDescriptorPrePass(extensionEntries, structure) {
+async function buildDescriptorPrePass(extensionEntries, structure, metadataSchema) {
   const { mapping, warnings } = buildDescriptorMapping(extensionEntries);
 
   for (const w of warnings) log.warn(`  descriptor_definition: ${w}`);
 
-  // Always create Unsupported/ — even if empty, users see at a glance what couldn't be mapped
-  await mkdir(EXTENSION_UNSUPPORTED_DIR, { recursive: true });
-  log.dim(`  ${EXTENSION_UNSUPPORTED_DIR}/`);
-
   // Create one sub-directory per mapped descriptor name
   for (const dirName of new Set(Object.values(mapping))) {
     const fullDir = `${EXTENSION_DESCRIPTORS_DIR}/${dirName}`;
@@ -1985,9 +2107,63 @@ async function buildDescriptorPrePass(extensionEntries, structure) {
   await saveDescriptorMapping(structure, mapping);
   log.dim(`  Saved descriptorMapping to .dbo/structure.json`);
 
+  // Parse form-control-code from descriptor_definition records → populate metadata_schema.json
+  const descriptorDefs = extensionEntries.filter(r =>
+    resolveFieldValue(r.Descriptor) === 'descriptor_definition'
+  );
+  let schemaUpdated = false;
+  for (const defRecord of descriptorDefs) {
+    const descriptor = resolveFieldValue(defRecord.String1);
+    if (!descriptor) continue;
+    const string5 = resolveFieldValue(defRecord.String5);
+    const { colToExt, colToTitle } = parseFormControlCode(string5);
+    if (colToExt.size === 0) continue;
+
+    // Only populate if no entry already exists for this descriptor
+    const existing = getTemplateCols(metadataSchema, 'extension', descriptor);
+    if (existing) continue;
+
+    const refEntries = [];
+    for (const [col, ext] of colToExt) {
+      const title = colToTitle.get(col);
+      const titleSuffix = title ? `{${title}}` : '';
+      refEntries.push(`${col}=@reference:@Name[${ext}]${titleSuffix}`);
+    }
+    setTemplateCols(metadataSchema, 'extension', descriptor, refEntries);
+    schemaUpdated = true;
+  }
+  if (schemaUpdated) await saveMetadataSchema(metadataSchema);
+
   return mapping;
 }
 
+/**
+ * Derive companion column @reference expressions from metadata_schema.json for a descriptor.
+ * Returns array of parsed @reference expression objects.
+ */
+function getDescriptorCompanionCols(metadataSchema, descriptor) {
+  const cols = getTemplateCols(metadataSchema, 'extension', descriptor) ?? [];
+  const refs = [];
+  for (const col of cols) {
+    const parsed = parseReferenceExpression(col);
+    if (parsed) refs.push(parsed);
+  }
+  return refs;
+}
+
+/**
+ * Derive filename column from metadata_schema.json @reference entries for a descriptor.
+ * Returns the column name (e.g., "Name") or falls back to 'Name'.
+ */
+function getDescriptorFilenameCol(metadataSchema, descriptor) {
+  const cols = getTemplateCols(metadataSchema, 'extension', descriptor) ?? [];
+  for (const col of cols) {
+    const parsed = parseReferenceExpression(col);
+    if (parsed?.filenameCol?.startsWith('@')) return parsed.filenameCol.slice(1);
+  }
+  return 'Name';
+}
+
 /**
  * Resolve filename column and content extraction preferences for one descriptor.
  * Prompts the user on first use; saves to config.json; respects -y and --force.
@@ -2006,16 +2182,16 @@ async function resolveDescriptorPreferences(descriptor, records, options) {
   let filenameCol;
   const savedCol = await loadDescriptorFilenamePreference(descriptor);
 
+  const defaultCol = columns.includes('Name') ? 'Name'
+    : columns.includes('UID') ? 'UID' : columns[0];
+
   if (options.yes) {
-    filenameCol = columns.includes('Name') ? 'Name'
-      : columns.includes('UID') ? 'UID' : columns[0];
-  } else if (savedCol && !options.force) {
+    filenameCol = defaultCol;
+  } else if (savedCol && !options.force && !options.configure) {
     filenameCol = savedCol;
     log.dim(`  Filename column for "${descriptor}": "${filenameCol}" (saved)`);
-  } else {
+  } else if (options.configure) {
     const inquirer = (await import('inquirer')).default;
-    const defaultCol = columns.includes('Name') ? 'Name'
-      : columns.includes('UID') ? 'UID' : columns[0];
     const { col } = await inquirer.prompt([{
       type: 'list', name: 'col',
       message: `Filename column for "${descriptor}" extensions:`,
@@ -2024,6 +2200,11 @@ async function resolveDescriptorPreferences(descriptor, records, options) {
     filenameCol = col;
     await saveDescriptorFilenamePreference(descriptor, filenameCol);
     log.dim(`  Saved filename column for "${descriptor}": "${filenameCol}"`);
+  } else {
+    // Auto-apply default silently
+    filenameCol = defaultCol;
+    await saveDescriptorFilenamePreference(descriptor, filenameCol);
+    log.dim(`  Using filename column "${filenameCol}" for "${descriptor}"`);
   }
 
   // ── Content extraction ───────────────────────────────────────────────
@@ -2052,12 +2233,11 @@ async function resolveDescriptorPreferences(descriptor, records, options) {
     }
 
     if (base64Cols.length > 0) {
-      const savedExtractions = options.force
+      const savedExtractions = (options.force || options.configure)
         ? null
         : await loadDescriptorContentExtractions(descriptor);
       const newPreferences = savedExtractions ? { ...savedExtractions } : {};
       let changed = false;
-      const inquirer = (await import('inquirer')).default;
 
       for (const { col, snippet } of base64Cols) {
         if (savedExtractions) {
@@ -2069,24 +2249,35 @@ async function resolveDescriptorPreferences(descriptor, records, options) {
             continue;
           }
         }
-        const preview = snippet ? ` ("${snippet}")` : '';
-        const { extract } = await inquirer.prompt([{
-          type: 'confirm', name: 'extract',
-          message: `Extract column "${col}" (${descriptor}) as companion file?${preview}`,
-          default: true,
-        }]);
-        if (extract) {
-          const guessed = guessExtensionForDescriptor(descriptor, col);
-          const { ext } = await inquirer.prompt([{
-            type: 'input', name: 'ext',
-            message: `File extension for "${col}" (${descriptor}):`,
-            default: guessed,
+
+        if (options.configure) {
+          // --configure: prompt user
+          const preview = snippet ? ` ("${snippet}")` : '';
+          const inquirer = (await import('inquirer')).default;
+          const { extract } = await inquirer.prompt([{
+            type: 'confirm', name: 'extract',
+            message: `Extract column "${col}" (${descriptor}) as companion file?${preview}`,
+            default: true,
           }]);
-          const cleanExt = ext.replace(/^\./, '');
-          contentColsToExtract.push({ col, ext: cleanExt });
-          newPreferences[col] = cleanExt;
+          if (extract) {
+            const guessed = guessExtensionForDescriptor(descriptor, col);
+            const { ext } = await inquirer.prompt([{
+              type: 'input', name: 'ext',
+              message: `File extension for "${col}" (${descriptor}):`,
+              default: guessed,
+            }]);
+            const cleanExt = ext.replace(/^\./, '');
+            contentColsToExtract.push({ col, ext: cleanExt });
+            newPreferences[col] = cleanExt;
+          } else {
+            newPreferences[col] = false;
+          }
         } else {
-          newPreferences[col] = false;
+          // Auto-apply: extract with guessed extension
+          const guessed = guessExtensionForDescriptor(descriptor, col);
+          contentColsToExtract.push({ col, ext: guessed });
+          newPreferences[col] = guessed;
+          log.dim(`  Auto-extracting "${col}" for "${descriptor}" as .${guessed}`);
         }
         changed = true;
       }
@@ -2116,24 +2307,30 @@ async function resolveDocumentationPlacement(options) {
   if (options.yes) return 'inline';
 
   const saved = await loadExtensionDocumentationMDPlacement();
-  if (saved && !options.force) {
+  if (saved && !options.force && !options.configure) {
     log.dim(`  Documentation MD placement: ${saved} (saved)`);
     return saved;
   }
 
-  const inquirer = (await import('inquirer')).default;
-  const { placement } = await inquirer.prompt([{
-    type: 'list', name: 'placement',
-    message: 'Where should extracted documentation MD files be placed?',
-    choices: [
-      { name: 'docs/<filename>.md  — project root (recommended)', value: 'root' },
-      { name: 'extension/documentation/<filename>.md  — inline alongside metadata', value: 'inline' },
-    ],
-    default: 'root',
-  }]);
+  let placement;
+  if (options.configure) {
+    const inquirer = (await import('inquirer')).default;
+    ({ placement } = await inquirer.prompt([{
+      type: 'list', name: 'placement',
+      message: 'Where should extracted documentation MD files be placed?',
+      choices: [
+        { name: 'docs/<filename>.md  — project root (recommended)', value: 'root' },
+        { name: 'extension/documentation/<filename>.md  — inline alongside metadata', value: 'inline' },
+      ],
+      default: 'root',
+    }]));
+  } else {
+    // Auto-apply default
+    placement = 'root';
+  }
 
   await saveExtensionDocumentationMDPlacement(placement);
-  log.dim(`  Saved documentation MD placement: "${placement}"`);
+  log.dim(`  Documentation MD placement: ${placement}`);
   return placement;
 }
 
@@ -2158,7 +2355,8 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
   log.info(`Processing ${entries.length} extension record(s)...`);
 
   // Step A: Pre-pass — build mapping + create directories
-  const mapping = await buildDescriptorPrePass(entries, structure);
+  const metadataSchema = await loadMetadataSchema();
+  const mapping = await buildDescriptorPrePass(entries, structure, metadataSchema);
 
   // Clear documentation preferences when --force is used with --documentation-only
   if (options.documentationOnly && options.force) {
@@ -2171,7 +2369,7 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
   // Step B: Group records by descriptor
   const groups = new Map(); // descriptor → { dir, records[] }
   for (const record of entries) {
-    const descriptor = record.Descriptor || '__unsupported__';
+    const descriptor = record.Descriptor || '__no_descriptor__';
 
     // --documentation-only: skip non-documentation records
     if (options.documentationOnly && descriptor !== 'documentation') continue;
@@ -2186,13 +2384,14 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
   const descriptorPrefs = new Map();
   let docPlacement = 'inline';
 
-  for (const [descriptor, { records }] of groups.entries()) {
-    if (descriptor === '__unsupported__') {
-      descriptorPrefs.set(descriptor, { filenameCol: null, contentColsToExtract: [] });
+  for (const [descriptor] of groups.entries()) {
+    if (descriptor === '__no_descriptor__') {
+      descriptorPrefs.set(descriptor, { filenameCol: 'Name', companionRefs: [] });
       continue;
     }
-    const prefs = await resolveDescriptorPreferences(descriptor, records, options);
-    descriptorPrefs.set(descriptor, prefs);
+    const filenameCol = getDescriptorFilenameCol(metadataSchema, descriptor);
+    const companionRefs = getDescriptorCompanionCols(metadataSchema, descriptor);
+    descriptorPrefs.set(descriptor, { filenameCol, companionRefs });
 
     if (descriptor === 'documentation') {
       docPlacement = await resolveDocumentationPlacement(options);
@@ -2210,9 +2409,9 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
   const config = await loadConfig();
 
   for (const [descriptor, { dir, records }] of groups.entries()) {
-    const { filenameCol, contentColsToExtract } = descriptorPrefs.get(descriptor);
+    const { filenameCol, companionRefs } = descriptorPrefs.get(descriptor);
     const useRootDoc = (descriptor === 'documentation' && docPlacement === 'root');
-    const mdColInfo = useRootDoc ? contentColsToExtract.find(c => c.ext === 'md') : null;
+    const mdColInfo = useRootDoc ? companionRefs.find(r => r.extensionCol === 'md') : null;
     const usedNames = new Map(); // name → count, for collision resolution within this descriptor group
 
     log.info(`Processing ${records.length} "${descriptor}" extension(s) → ${dir}/`);
@@ -2284,11 +2483,11 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
       }
 
       // Check if any @reference content files are missing — force re-extraction if so
-      let hasNewExtractions = contentColsToExtract.length > 0;
+      let hasNewExtractions = companionRefs.length > 0;
       if (!hasNewExtractions && await fileExists(metaPath)) {
         try {
           const existingMeta = JSON.parse(await readFile(metaPath, 'utf8'));
-          for (const col of (existingMeta._contentColumns || [])) {
+          for (const col of (existingMeta._companionReferenceColumns || existingMeta._contentColumns || [])) {
             const ref = existingMeta[col];
             if (ref && String(ref).startsWith('@')) {
               const refName = String(ref).substring(1);
@@ -2348,19 +2547,23 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
       for (const [key, value] of Object.entries(record)) {
         if (key === 'children') continue;
 
-        const extractInfo = contentColsToExtract.find(c => c.col === key);
-        if (extractInfo) {
+        const companionRef = companionRefs.find(r => r.column.toLowerCase() === key.toLowerCase());
+        if (companionRef) {
           const isBase64 = value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64';
           const decoded = isBase64 ? (resolveContentValue(value) ?? '') : (value ?? '');
           let colFilePath, refValue;
+          const ext = companionRef.extensionCol?.startsWith('@')
+            ? (String(record[companionRef.extensionCol.slice(1)] ?? '') || companionRef.extensionFallback || 'txt').toLowerCase()
+            : (companionRef.extensionCol || companionRef.extensionFallback || 'txt');
 
-          if (mdColInfo && extractInfo.col === mdColInfo.col) {
+          if (mdColInfo && companionRef.column === mdColInfo.column) {
             // Root placement: docs/<name>.md (natural name, no ~UID)
             const docFileName = `${name}.md`;
             colFilePath = join(DOCUMENTATION_DIR, docFileName);
             refValue = `@/${DOCUMENTATION_DIR}/${docFileName}`;
           } else {
-            const colFileName = `${name}.${key}.${extractInfo.ext}`;
+            const colSegment = companionRef.title ? sanitizeFilename(companionRef.title) : key;
+            const colFileName = `${name}.${colSegment}.${ext}`;
             colFilePath = join(dir, colFileName);
             refValue = `@${colFileName}`;
           }
@@ -2385,7 +2588,7 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
       }
 
       meta._entity = 'extension';
-      if (extractedCols.length > 0) meta._contentColumns = extractedCols;
+      if (extractedCols.length > 0) meta._companionReferenceColumns = extractedCols;
 
       await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
       if (serverTz) {
@@ -2397,15 +2600,15 @@ async function processExtensionEntries(entries, structure, options, serverTz) {
 
     // --- Seed metadata template for this descriptor ---
     if (records.length > 0) {
-      const templates = await loadMetadataTemplates();
+      const templates = await loadMetadataSchema();
       if (templates !== null) {
         const existing = getTemplateCols(templates, 'extension', descriptor);
         if (!existing) {
           const firstRecord = records[0];
-          const extractedCols = contentColsToExtract.map(c => c.col);
+          const extractedCols = companionRefs.map(r => r.column);
           const cols = buildTemplateFromCloneRecord(firstRecord, extractedCols);
           setTemplateCols(templates, 'extension', descriptor, cols);
-          await saveMetadataTemplates(templates);
+          await saveMetadataSchema(templates);
         }
       }
     }
@@ -2688,8 +2891,27 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
         error: err.message,
       });
 
+      // For 404s, write metadata anyway so the file isn't re-prompted on next clone.
+      // The media file is genuinely gone from the server — nothing to download.
+      if (is404) {
+        const staleMeta = {};
+        for (const [key, value] of Object.entries(record)) {
+          if (key === 'children') continue;
+          staleMeta[key] = value;
+        }
+        staleMeta._entity = 'media';
+        staleMeta._stale404 = true;
+        try {
+          await writeFile(metaPath, JSON.stringify(staleMeta, null, 2) + '\n');
+          if (serverTz && (record._CreatedOn || record._LastUpdated)) {
+            await setFileTimestamps(metaPath, record._CreatedOn, record._LastUpdated, serverTz);
+          }
+          refs.push({ uid: record.UID, metaPath });
+        } catch { /* non-critical */ }
+      }
+
       failed++;
-      continue; // Skip metadata if download failed
+      continue; // Skip media write if download failed
     }
 
     // Build metadata
@@ -2788,7 +3010,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
       const raw = await readFile(probeMeta, 'utf8');
       const localMeta = JSON.parse(raw);
       // Extract extension from Content @reference (e.g. "@Name~uid.html")
-      for (const col of (localMeta._contentColumns || ['Content'])) {
+      for (const col of (localMeta._companionReferenceColumns || localMeta._contentColumns || ['Content'])) {
         const ref = localMeta[col];
         if (typeof ref === 'string' && ref.startsWith('@')) {
           const refExt = extname(ref);
@@ -2911,7 +3133,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
       const configWithTz = { ...config, ServerTimezone: serverTz };
       const localSyncTime = await getLocalSyncTime(metaPath);
 
-      // If local metadata has no _LastUpdated (e.g. from dbo add with incomplete fields),
+      // If local metadata has no _LastUpdated (e.g. from dbo adopt with incomplete fields),
       // always treat as server-newer so pull populates missing columns.
       let localMissingLastUpdated = false;
       try {
@@ -2923,7 +3145,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
       const serverDate = parseServerDate(record._LastUpdated, serverTz);
 
       if (serverNewer) {
-        // Incomplete metadata (no _LastUpdated) from dbo add — auto-accept without prompting
+        // Incomplete metadata (no _LastUpdated) from dbo adopt — auto-accept without prompting
         if (localMissingLastUpdated) {
           log.dim(`  Completing metadata: ${fileName}`);
           // Fall through to write
@@ -3014,8 +3236,8 @@ async function processRecord(entityName, record, structure, options, usedNames,
         await writeFile(colFilePath, decoded);
         await upsertDeployEntry(colFilePath, record.UID, entityName, key);
         meta[key] = `@${colFileName}`;
-        if (!meta._contentColumns) meta._contentColumns = [];
-        meta._contentColumns.push(key);
+        if (!meta._companionReferenceColumns) meta._companionReferenceColumns = [];
+        meta._companionReferenceColumns.push(key);
       } else {
         meta[key] = decoded;
       }
@@ -3025,8 +3247,8 @@ async function processRecord(entityName, record, structure, options, usedNames,
   }
 
   meta._entity = entityName;
-  if (hasContent && !meta._contentColumns) {
-    meta._contentColumns = ['Content'];
+  if (hasContent && !meta._companionReferenceColumns) {
+    meta._companionReferenceColumns = ['Content'];
   }
 
   // Extension was derived from Name/Path for local filename purposes only.
@@ -3195,20 +3417,13 @@ async function resolveOutputFilenameColumns(appJson, options) {
   const result = {};
 
   for (const entityKey of OUTPUT_HIERARCHY_ENTITIES) {
-    // Check saved preference
-    const saved = await loadOutputFilenamePreference(entityKey);
+    // Check saved preference (skip if --configure to allow re-prompting)
+    const saved = options.configure ? null : await loadOutputFilenamePreference(entityKey);
     if (saved) {
       result[entityKey] = saved;
       continue;
     }
 
-    // In -y mode use defaults
-    if (options.yes) {
-      result[entityKey] = defaults[entityKey];
-      await saveOutputFilenamePreference(entityKey, defaults[entityKey]);
-      continue;
-    }
-
     // Find a sample record to get available columns
     // Check top-level first, then look inside nested output children
     let records = appJson.children[entityKey] || [];
@@ -3235,18 +3450,25 @@ async function resolveOutputFilenameColumns(appJson, options) {
       if (columns.includes(fb)) { defaultCol = fb; break; }
     }
 
-    const inquirer = (await import('inquirer')).default;
-    const { col } = await inquirer.prompt([{
-      type: 'list',
-      name: 'col',
-      message: `Which column should be used as the filename for ${docNames[entityKey]} (${entityKey}) records?`,
-      choices: columns,
-      default: defaultCol,
-    }]);
+    let col;
+    if (options.configure) {
+      // --configure: prompt user to pick
+      const inquirer = (await import('inquirer')).default;
+      ({ col } = await inquirer.prompt([{
+        type: 'list',
+        name: 'col',
+        message: `Which column should be used as the filename for ${docNames[entityKey]} (${entityKey}) records?`,
+        choices: columns,
+        default: defaultCol,
+      }]));
+    } else {
+      // Auto-apply best default silently
+      col = defaultCol;
+    }
 
     result[entityKey] = col;
     await saveOutputFilenamePreference(entityKey, col);
-    log.dim(`  Saved filename column preference for ${entityKey}`);
+    log.dim(`  Using filename column "${col}" for ${entityKey}`);
   }
 
   return result;
@@ -3336,9 +3558,9 @@ async function extractCustomSqlIfNeeded(entityObj, companionStem, outputDir, ser
   await writeFile(companionPath, hasContent ? decoded : '', 'utf8');
   await upsertDeployEntry(companionPath, entityObj.UID || entityObj._uid, 'output', 'CustomSQL');
   entityObj.CustomSQL = `@${companionName}`;
-  entityObj._contentColumns = entityObj._contentColumns || [];
-  if (!entityObj._contentColumns.includes('CustomSQL')) {
-    entityObj._contentColumns.push('CustomSQL');
+  entityObj._companionReferenceColumns = entityObj._companionReferenceColumns || [];
+  if (!entityObj._companionReferenceColumns.includes('CustomSQL')) {
+    entityObj._companionReferenceColumns.push('CustomSQL');
   }
 
   // Sync timestamps