@dboio/cli 0.9.8 → 0.10.1

package/src/commands/clone.js

@@ -1,6 +1,7 @@
 import { Command } from 'commander';
-import { readFile, writeFile, mkdir, access } from 'fs/promises';
-import { join, basename, extname } from 'path';
+import { readFile, writeFile, mkdir, access, readdir, rename } from 'fs/promises';
+import { join, basename, extname, dirname } from 'path';
+import { fileURLToPath } from 'url';
 import { DboClient } from '../lib/client.js';
 import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference, saveEntityContentExtractions, loadEntityContentExtractions, saveAppJsonBaseline, addDeleteEntry, loadCollisionResolutions, saveCollisionResolutions, loadSynchronize, saveSynchronize, saveAppModifyKey, loadTransactionKeyPreset, saveTransactionKeyPreset, loadOutputFilenamePreference, saveOutputFilenamePreference, saveCloneSource, loadCloneSource, saveDescriptorFilenamePreference, loadDescriptorFilenamePreference, saveDescriptorContentExtractions, loadDescriptorContentExtractions, saveExtensionDocumentationMDPlacement, loadExtensionDocumentationMDPlacement } from '../lib/config.js';
 import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_NAMES, OUTPUT_ENTITY_MAP, OUTPUT_HIERARCHY_ENTITIES, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, DOCUMENTATION_DIR, buildDescriptorMapping, saveDescriptorMapping, loadDescriptorMapping, resolveExtensionSubDir } from '../lib/structure.js';
@@ -9,6 +10,7 @@ import { buildUidFilename, detectLegacyDotUid } from '../lib/filenames.js';
 import { setFileTimestamps, parseServerDate } from '../lib/timestamps.js';
 import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge, isDiffable } from '../lib/diff.js';
 import { checkDomainChange } from '../lib/domain-guard.js';
+import { applyTrashIcon, ensureTrashIcon } from '../lib/folder-icon.js';
 import { loadMetadataTemplates, saveMetadataTemplates, getTemplateCols, setTemplateCols, buildTemplateFromCloneRecord } from '../lib/metadata-templates.js';
 
 /**
@@ -93,7 +95,7 @@ function resolvePathToBinsDir(pathValue, structure) {
  * Extract path components for content/generic records (read-only, no file writes).
  * Replicates logic from processRecord() for collision detection.
  */
-function resolveRecordPaths(entityName, record, structure, placementPref) {
+export function resolveRecordPaths(entityName, record, structure, placementPref) {
   let name = sanitizeFilename(String(record.Name || record.UID || 'untitled'));
 
   // Determine extension (priority: Extension field > Name > Path)
@@ -152,7 +154,7 @@ function resolveRecordPaths(entityName, record, structure, placementPref) {
  * Extract path components for media records.
  * Replicates logic from processMediaEntries() for collision detection.
  */
-function resolveMediaPaths(record, structure, placementPref) {
+export function resolveMediaPaths(record, structure, placementPref) {
   const filename = record.Filename || `${record.Name || record.UID}.${(record.Extension || 'bin').toLowerCase()}`;
   const name = sanitizeFilename(filename.replace(/\.[^.]+$/, ''));
   const ext = (record.Extension || 'bin').toLowerCase();
@@ -191,7 +193,7 @@ function resolveMediaPaths(record, structure, placementPref) {
  * Extract path components for entity-dir records.
  * Simplified from processEntityDirEntries() for collision detection.
  */
-function resolveEntityDirPaths(entityName, record, dirName) {
+export function resolveEntityDirPaths(entityName, record, dirName) {
   let name;
   if (entityName === 'app_version' && record.Number) {
     name = sanitizeFilename(String(record.Number));
@@ -644,10 +646,10 @@ export async function performClone(source, options = {}) {
   const effectiveDomain = options.domain || config.domain;
   let appJson;
 
-  // Step 1: Source mismatch detection
+  // Step 1: Source mismatch detection (skip in pull mode)
   // Warn when the user provides an explicit source that differs from the stored one.
-  const storedCloneSource = await loadCloneSource();
-  if (source && storedCloneSource && source !== storedCloneSource) {
+  const storedCloneSource = options.pullMode ? null : await loadCloneSource();
+  if (!options.pullMode && source && storedCloneSource && source !== storedCloneSource) {
     if (!options.force && !options.yes) {
       log.warn('');
       log.warn(`  ⚠  This project was previously cloned from: ${storedCloneSource}`);
@@ -707,95 +709,132 @@ export async function performClone(source, options = {}) {
     }
   }
 
-  log.success(`Cloning "${appJson.Name}" (${appJson.ShortName})`);
+  log.success(`${options.pullMode ? 'Pulling' : 'Cloning'} "${appJson.Name}" (${appJson.ShortName})`);
 
-  // Check for un-pushed staged items in synchronize.json
-  await checkPendingSynchronize(options);
+  // Check for un-pushed staged items in synchronize.json (skip in pull mode)
+  if (!options.pullMode) {
+    await checkPendingSynchronize(options);
+  }
 
   // Ensure sensitive files are gitignored
-  await ensureGitignore(['.dbo/credentials.json', '.dbo/cookies.txt']);
-
-  // Step 2: Update .dbo/config.json
-  await updateConfigWithApp({
-    AppID: appJson.AppID,
-    AppUID: appJson.UID,
-    AppName: appJson.Name,
-    AppShortName: appJson.ShortName,
-  });
-  await saveCloneSource(activeSource || 'default');
-  log.dim('  Updated .dbo/config.json with app metadata');
-
-  // Detect and store ModifyKey for locked/production apps
-  const modifyKey = appJson.ModifyKey || null;
-  await saveAppModifyKey(modifyKey);
-  if (modifyKey) {
-    log.warn('');
-    log.warn('  ⚠  This app has a ModifyKey set (production/locked mode).');
-    log.warn('     You will be prompted to enter the ModifyKey before any push, input, add, content deploy, or deploy command.');
-    log.warn('');
-  }
-
-  // Prompt for TransactionKeyPreset if not already set
-  const existingPreset = await loadTransactionKeyPreset();
-  if (!existingPreset) {
-    if (options.yes || !process.stdin.isTTY) {
-      await saveTransactionKeyPreset('RowUID');
-      log.dim('  TransactionKeyPreset: RowUID (default)');
-    } else {
-      const inquirer = (await import('inquirer')).default;
-      const { preset } = await inquirer.prompt([{
-        type: 'list',
-        name: 'preset',
-        message: 'Which row key should the CLI use when building input expressions?',
-        choices: [
-          { name: 'RowUID (recommended — stable across domains)', value: 'RowUID' },
-          { name: 'RowID (numeric IDs)', value: 'RowID' },
-        ],
-      }]);
-      await saveTransactionKeyPreset(preset);
-      log.dim(`  TransactionKeyPreset: ${preset}`);
+  await ensureGitignore(['.dbo/credentials.json', '.dbo/cookies.txt', '.dbo/.app_baseline.json']);
+
+  // Step 2: Update .dbo/config.json (skip in pull mode — config already set)
+  if (!options.pullMode) {
+    await updateConfigWithApp({
+      AppID: appJson.AppID,
+      AppUID: appJson.UID,
+      AppName: appJson.Name,
+      AppShortName: appJson.ShortName,
+    });
+    await saveCloneSource(activeSource || 'default');
+    log.dim('  Updated .dbo/config.json with app metadata');
+  }
+
+  // Detect and store ModifyKey for locked/production apps (skip in pull mode)
+  if (!options.pullMode) {
+    const modifyKey = appJson.ModifyKey || null;
+    await saveAppModifyKey(modifyKey);
+    if (modifyKey) {
+      log.warn('');
+      log.warn('  ⚠  This app has a ModifyKey set (production/locked mode).');
+      log.warn('     You will be prompted to enter the ModifyKey before any push, input, add, content deploy, or deploy command.');
+      log.warn('');
     }
   }
 
-  // Step 3: Update package.json
-  await updatePackageJson(appJson, config);
+  // Prompt for TransactionKeyPreset if not already set (skip in pull mode)
+  if (!options.pullMode) {
+    const existingPreset = await loadTransactionKeyPreset();
+    if (!existingPreset) {
+      if (options.yes || !process.stdin.isTTY) {
+        await saveTransactionKeyPreset('RowUID');
+        log.dim('  TransactionKeyPreset: RowUID (default)');
+      } else {
+        const inquirer = (await import('inquirer')).default;
+        const { preset } = await inquirer.prompt([{
+          type: 'list',
+          name: 'preset',
+          message: 'Which row key should the CLI use when building input expressions?',
+          choices: [
+            { name: 'RowUID (recommended — stable across domains)', value: 'RowUID' },
+            { name: 'RowID (numeric IDs)', value: 'RowID' },
+          ],
+        }]);
+        await saveTransactionKeyPreset(preset);
+        log.dim(`  TransactionKeyPreset: ${preset}`);
+      }
+    }
+  }
 
-  // Step 4: Create default project directories + bin structure
-  for (const dir of DEFAULT_PROJECT_DIRS) {
-    await mkdir(dir, { recursive: true });
+  // Step 3: Update package.json (skip in pull mode)
+  if (!options.pullMode) {
+    await updatePackageJson(appJson, config);
   }
 
+  // Step 4: Create default project directories + bin structure
   const bins = appJson.children.bin || [];
   const structure = buildBinHierarchy(bins, appJson.AppID);
-  const createdDirs = await createDirectories(structure);
-  await saveStructureFile(structure);
-
-  const totalDirs = DEFAULT_PROJECT_DIRS.length + createdDirs.length;
-  log.success(`Created ${totalDirs} director${totalDirs === 1 ? 'y' : 'ies'}`);
-  for (const d of DEFAULT_PROJECT_DIRS) log.dim(`  ${d}/`);
-  for (const d of createdDirs) log.dim(`  ${d}/`);
-
-  // Warn about legacy mixed-case directories from pre-0.9.1
-  const LEGACY_DIR_MAP = {
-    'Bins':        'bins',
-    'Automations': 'automation',
-    'App Versions': 'app_version',
-    'Documentation': 'docs',
-    'Sites':       'site',
-    'Extensions':  'extension',
-    'Data Sources': 'data_source',
-    'Groups':      'group',
-    'Integrations': 'integration',
-    'Trash':       'trash',
-    'Src':         'src',
-  };
-  for (const [oldName, newName] of Object.entries(LEGACY_DIR_MAP)) {
-    try {
-      await access(join(process.cwd(), oldName));
-      log.warn(`Legacy directory detected: "${oldName}/" — rename it to "${newName}/" for the new convention.`);
-    } catch {
-      // does not exist — no warning needed
+
+  if (!options.pullMode) {
+    for (const dir of DEFAULT_PROJECT_DIRS) {
+      await mkdir(dir, { recursive: true });
+    }
+
+    // Create media sub-directories for this app:
+    //   media/<ShortName>/app/  — app-level media assets
+    //   media/<ShortName>/user/ — user-uploaded media
+    const appShortName = appJson.ShortName;
+    const mediaSubs = [];
+    if (appShortName) {
+      const mediaDirs = [
+        `media/${appShortName}/app`,
+        `media/${appShortName}/user`,
+      ];
+      for (const sub of mediaDirs) {
+        await mkdir(sub, { recursive: true });
+        mediaSubs.push(sub);
+      }
     }
+
+    // Best-effort: apply trash icon
+    await applyTrashIcon(join(process.cwd(), 'trash'));
+
+    const createdDirs = await createDirectories(structure);
+    await saveStructureFile(structure);
+
+    const totalDirs = DEFAULT_PROJECT_DIRS.length + mediaSubs.length + createdDirs.length;
+    log.success(`Created ${totalDirs} director${totalDirs === 1 ? 'y' : 'ies'}`);
+    for (const d of DEFAULT_PROJECT_DIRS) log.dim(`  ${d}/`);
+    for (const d of mediaSubs) log.dim(`  ${d}/`);
+    for (const d of createdDirs) log.dim(`  ${d}/`);
+
+    // Warn about legacy mixed-case directories from pre-0.9.1
+    const LEGACY_DIR_MAP = {
+      'Bins':        'bins',
+      'Automations': 'automation',
+      'App Versions': 'app_version',
+      'Documentation': 'docs',
+      'Sites':       'site',
+      'Extensions':  'extension',
+      'Data Sources': 'data_source',
+      'Groups':      'group',
+      'Integrations': 'integration',
+      'Trash':       'trash',
+      'Src':         'src',
+    };
+    for (const [oldName, newName] of Object.entries(LEGACY_DIR_MAP)) {
+      try {
+        await access(join(process.cwd(), oldName));
+        log.warn(`Legacy directory detected: "${oldName}/" — rename it to "${newName}/" for the new convention.`);
+      } catch {
+        // does not exist — no warning needed
+      }
+    }
+  } else {
+    // Pull mode: reuse existing structure, just ensure dirs exist for new bins
+    await createDirectories(structure);
+    await saveStructureFile(structure);
   }
 
   // Step 4b: Determine placement preferences (from config or prompt)
@@ -816,9 +855,9 @@ export async function performClone(source, options = {}) {
     log.info(`Entity filter: only processing ${options.entity}`);
   }
 
-  // Step 4c: Detect and resolve file path collisions (skip in entity-filter mode)
+  // Step 4c: Detect and resolve file path collisions (skip in pull mode and entity-filter mode)
   let toDeleteUIDs = new Set();
-  if (!entityFilter) {
+  if (!options.pullMode && !entityFilter) {
     log.info('Scanning for file path collisions...');
     const fileRegistry = await buildFileRegistry(appJson, structure, placementPrefs);
     toDeleteUIDs = await resolveCollisions(fileRegistry, options);
@@ -841,6 +880,11 @@ export async function performClone(source, options = {}) {
     );
   }
 
+  // Step 5a: Write manifest.json to project root (from server content or resolved template)
+  if (!entityFilter || entityFilter.has('content')) {
+    await writeManifestJson(appJson, contentRefs);
+  }
+
   // Step 5b: Process media → download binary files + metadata (skip rejected records)
   let mediaRefs = [];
   if (!entityFilter || entityFilter.has('media')) {
@@ -900,18 +944,18 @@ export async function performClone(source, options = {}) {
   // Step 7: Save app.json with references
   await saveAppJson(appJson, contentRefs, otherRefs, effectiveDomain);
 
-  // Step 8: Create .app.json baseline for delta tracking (skip in entity-filter mode to avoid overwriting)
+  // Step 8: Create .dbo/.app_baseline.json baseline for delta tracking (skip in entity-filter mode to avoid overwriting)
   if (!entityFilter) {
     await saveBaselineFile(appJson);
   }
 
-  // Step 9: Ensure .app.json is in .gitignore
-  await ensureGitignore(['.app.json']);
-
   log.plain('');
-  log.success(entityFilter ? `Clone complete! (filtered: ${options.entity})` : 'Clone complete!');
+  const verb = options.pullMode ? 'Pull' : 'Clone';
+  log.success(entityFilter ? `${verb} complete! (filtered: ${options.entity})` : `${verb} complete!`);
   log.dim('  app.json saved to project root');
-  log.dim('  Run "dbo login" to authenticate, then "dbo push" to deploy changes');
+  if (!options.pullMode) {
+    log.dim('  Run "dbo login" to authenticate, then "dbo push" to deploy changes');
+  }
 }
 
 /**
@@ -922,7 +966,7 @@ export async function performClone(source, options = {}) {
  * Entity-dir names (e.g. "extension", "site") are matched directly.
  * Documentation aliases are also accepted (e.g. "column" → "output_value").
  */
-function resolveEntityFilter(entityArg) {
+export function resolveEntityFilter(entityArg) {
   if (!entityArg) return null;
 
   const input = entityArg.toLowerCase().trim();
@@ -947,6 +991,14 @@ async function resolvePlacementPreferences(appJson, options) {
   let contentPlacement = saved.contentPlacement;
   let mediaPlacement = saved.mediaPlacement;
 
+  // Pull mode: use saved preferences or default to 'bin', no prompts
+  if (options.pullMode) {
+    return {
+      contentPlacement: contentPlacement || 'bin',
+      mediaPlacement: mediaPlacement || 'bin',
+    };
+  }
+
   // --media-placement flag takes precedence over saved config
   if (options.mediaPlacement) {
     mediaPlacement = options.mediaPlacement === 'fullpath' ? 'fullpath' : 'bin';
@@ -1015,6 +1067,8 @@ async function resolvePlacementPreferences(appJson, options) {
 
 /**
  * Fetch app JSON from the server by AppShortName.
+ * Distinguishes between authentication failures (expired session) and
+ * genuine "app not found" responses, offering re-login when appropriate.
  */
 async function fetchAppFromServer(appShortName, options, config) {
   const client = new DboClient({ domain: options.domain, verbose: options.verbose });
@@ -1030,6 +1084,58 @@ async function fetchAppFromServer(appShortName, options, config) {
     throw err;
   }
 
+  // Check for authentication / session errors before parsing app data.
+  // The server may return HTTP 401/403 or a 200 envelope with Successful=false
+  // and messages containing user identity patterns.
+  const AUTH_PATTERNS = ['LoggedInUser_UID', 'LoggedInUserID', 'CurrentUserID', 'UserID', 'not authenticated', 'session expired', 'login required'];
+  const messages = result.messages || [];
+  const allMsgText = messages.filter(m => typeof m === 'string').join(' ');
+  const isAuthError = !result.ok && (result.status === 401 || result.status === 403)
+    || (!result.successful && AUTH_PATTERNS.some(p => allMsgText.includes(p)));
+
+  if (isAuthError) {
+    spinner.fail('Session expired or not authenticated');
+    log.warn('Your session appears to have expired.');
+    if (allMsgText) log.dim(`  Server: ${allMsgText.substring(0, 200)}`);
+
+    // Offer re-login
+    if (process.stdin.isTTY) {
+      const inquirer = (await import('inquirer')).default;
+      const { action } = await inquirer.prompt([{
+        type: 'list',
+        name: 'action',
+        message: 'How would you like to proceed?',
+        choices: [
+          { name: 'Re-login now (recommended)', value: 'relogin' },
+          { name: 'Abort', value: 'abort' },
+        ],
+      }]);
+
+      if (action === 'relogin') {
+        const { performLogin } = await import('./login.js');
+        await performLogin(options.domain || config.domain);
+        log.info('Retrying app fetch...');
+        return fetchAppFromServer(appShortName, options, config);
+      }
+    } else {
+      log.dim('  Run "dbo login" to authenticate, then retry.');
+    }
+    throw new Error('Authentication required. Run "dbo login" first.');
+  }
+
+  // Check for non-auth server errors (500, envelope Successful=false, etc.)
+  if (!result.ok && result.status >= 500) {
+    spinner.fail(`Server error (HTTP ${result.status})`);
+    if (allMsgText) log.dim(`  Server: ${allMsgText.substring(0, 200)}`);
+    throw new Error(`Server error (HTTP ${result.status}) fetching app "${appShortName}"`);
+  }
+
+  if (!result.successful && allMsgText) {
+    spinner.fail(`Server returned an error`);
+    log.warn(`  ${allMsgText.substring(0, 300)}`);
+    throw new Error(`Server error fetching app "${appShortName}": ${allMsgText.substring(0, 200)}`);
+  }
+
   const data = result.payload || result.data;
 
   // Handle all response shapes:
@@ -1100,6 +1206,21 @@ async function updatePackageJson(appJson, config) {
     changed = true;
   }
 
+  // Add @dboio/cli to devDependencies with current CLI version
+  if (!pkg.devDependencies || !pkg.devDependencies['@dboio/cli']) {
+    try {
+      const cliRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+      const cliPkg = JSON.parse(await readFile(join(cliRoot, 'package.json'), 'utf8'));
+      if (cliPkg.version) {
+        if (!pkg.devDependencies) pkg.devDependencies = {};
+        pkg.devDependencies['@dboio/cli'] = `^${cliPkg.version}`;
+        changed = true;
+      }
+    } catch {
+      // Could not read CLI version — skip
+    }
+  }
+
   if (changed) {
     await writeFile(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
     log.dim('  Updated package.json with app metadata');
@@ -1555,6 +1676,20 @@ async function buildDescriptorPrePass(extensionEntries, structure) {
     log.dim(`  ${fullDir}/`);
   }
 
+  // Create directories for descriptors not in the mapping but with a non-empty value
+  const unmappedDescriptors = new Set();
+  for (const rec of extensionEntries) {
+    const d = rec.Descriptor;
+    if (d && d !== 'descriptor_definition' && !mapping[d]) {
+      unmappedDescriptors.add(d);
+    }
+  }
+  for (const dirName of unmappedDescriptors) {
+    const fullDir = `${EXTENSION_DESCRIPTORS_DIR}/${dirName}`;
+    await mkdir(fullDir, { recursive: true });
+    log.dim(`  ${fullDir}/`);
+  }
+
   await saveDescriptorMapping(structure, mapping);
   log.dim(`  Saved descriptorMapping to .dbo/structure.json`);
 
@@ -1961,6 +2096,42 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
   // Track stale records (404s) for cleanup prompt
   const staleRecords = [];
 
+  // Pre-scan: determine which media files actually need downloading
+  // (new files or files with newer server timestamps)
+  const needsDownload = [];
+  const upToDateRefs = [];
+
+  for (const record of mediaRecords) {
+    if (skipUIDs.has(record.UID)) continue;
+
+    const { metaPath: scanMetaPath } = resolveMediaPaths(record, structure, mediaPlacement === 'fullpath' ? 'path' : mediaPlacement === 'bin' ? 'bin' : null);
+    const scanExists = await fileExists(scanMetaPath);
+
+    if (!scanExists) {
+      // New file — always needs download
+      needsDownload.push(record);
+    } else if (options.force) {
+      // Force mode — re-download everything
+      needsDownload.push(record);
+    } else {
+      // Existing file — check if server is newer
+      const configWithTz = { ...config, ServerTimezone: serverTz };
+      const localSyncTime = await getLocalSyncTime(scanMetaPath);
+      const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, configWithTz);
+      if (serverNewer) {
+        needsDownload.push(record);
+      } else {
+        // Up to date — still need ref for app.json
+        upToDateRefs.push({ uid: record.UID, metaPath: scanMetaPath });
+      }
+    }
+  }
+
+  if (needsDownload.length === 0) {
+    log.dim(`  All ${mediaRecords.length} media file(s) up to date`);
+    return upToDateRefs;
+  }
+
   // Determine if we can download (need a server connection)
   let canDownload = false;
   let client = null;
@@ -1970,7 +2141,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     const { download } = await inquirer.prompt([{
       type: 'confirm',
       name: 'download',
-      message: `${mediaRecords.length} media file(s) need to be downloaded from the server. Attempt download now?`,
+      message: `${needsDownload.length} media file(s) need to be downloaded (${mediaRecords.length - needsDownload.length} up to date). Attempt download now?`,
       default: true,
     }]);
     canDownload = download;
@@ -1989,8 +2160,8 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
   }
 
   if (!canDownload) {
-    log.warn(`Skipping ${mediaRecords.length} media file(s) — download not attempted`);
-    return [];
+    log.warn(`Skipping ${needsDownload.length} media file(s) — download not attempted`);
+    return upToDateRefs;
   }
 
   const refs = [];
@@ -2229,7 +2400,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     refs.push({ uid: record.UID, metaPath });
   }
 
-  log.info(`Media: ${downloaded} downloaded, ${failed} failed`);
+  log.info(`Media: ${downloaded} downloaded, ${failed} failed, ${upToDateRefs.length} up to date`);
 
   // Prompt for stale record cleanup
   if (staleRecords.length > 0 && !options.yes) {
@@ -2265,7 +2436,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     log.info(`Non-interactive mode: skipping stale cleanup for ${staleRecords.length} record(s)`);
   }
 
-  return refs;
+  return [...upToDateRefs, ...refs];
 }
 
 /**
@@ -2789,6 +2960,184 @@ export function buildOutputFilename(entityType, node, filenameCol, parentChain =
   return allSegments.join('.');
 }
 
+// ─── Inline Output Helpers ─────────────────────────────────────────────────
+
+const INLINE_DOC_NAMES = { output_value: 'column', output_value_filter: 'filter', output_value_entity_column_rel: 'join' };
+const INLINE_DOC_TO_PHYSICAL = { column: 'output_value', join: 'output_value_entity_column_rel', filter: 'output_value_filter' };
+const INLINE_DOC_KEYS = ['column', 'join', 'filter'];
+
+/**
+ * Build the companion file stem for a child entity within a root output file.
+ * e.g. root stem "_output~Sales~abc", entity "output_value", uid "col1"
+ *   → "_output~Sales~abc.column~col1"
+ *
+ * @param {string} rootStem - Root output file stem (no extension)
+ * @param {string} physicalEntity - Physical entity name ('output_value', etc.)
+ * @param {string} uid - Child entity UID
+ * @param {string} [parentChainStem] - Already-built ancestor stem (for nested children)
+ * @returns {string}
+ */
+export function getChildCompanionStem(rootStem, physicalEntity, uid, parentChainStem = rootStem) {
+  const docName = INLINE_DOC_NAMES[physicalEntity] || physicalEntity;
+  return `${parentChainStem}.${docName}~${uid}`;
+}
+
+/**
+ * Extract CustomSQL as a companion .sql file if rules require it.
+ * Rules:
+ *  1. Type === 'CustomSQL' → always extract (even empty)
+ *  2. Type !== 'CustomSQL' AND CustomSQL non-empty decoded value → extract
+ *  3. Otherwise → store "" inline; no file
+ *
+ * Mutates entityObj.CustomSQL to the @basename reference when extracted.
+ * Returns the companion filename (without directory) if written, else null.
+ *
+ * @param {Object} entityObj - The entity object (mutated in place)
+ * @param {string} companionStem - Stem for the companion file (no extension)
+ * @param {string} outputDir - Directory where the root output JSON lives
+ * @param {string} serverTz - Server timezone for timestamp syncing
+ * @returns {Promise<string|null>} - Companion filename or null
+ */
+async function extractCustomSqlIfNeeded(entityObj, companionStem, outputDir, serverTz) {
+  const rawSql = entityObj.CustomSQL;
+  const isCustomSqlType = entityObj.Type === 'CustomSQL';
+
+  // Decode base64 server value if needed
+  const decoded = resolveContentValue(rawSql) ?? '';
+  const hasContent = typeof decoded === 'string' && decoded.trim().length > 0;
+
+  if (!isCustomSqlType && !hasContent) {
+    // Rule 3: store empty string inline, no file
+    entityObj.CustomSQL = '';
+    return null;
+  }
+
+  // Rules 1 and 2: extract as companion .sql file
+  const companionName = `${companionStem}.CustomSQL.sql`;
+  const companionPath = join(outputDir, companionName);
+  await writeFile(companionPath, hasContent ? decoded : '', 'utf8');
+  entityObj.CustomSQL = `@${companionName}`;
+  entityObj._contentColumns = entityObj._contentColumns || [];
+  if (!entityObj._contentColumns.includes('CustomSQL')) {
+    entityObj._contentColumns.push('CustomSQL');
+  }
+
+  // Sync timestamps
+  if (serverTz && (entityObj._CreatedOn || entityObj._LastUpdated)) {
+    try { await setFileTimestamps(companionPath, entityObj._CreatedOn, entityObj._LastUpdated, serverTz); } catch { /* non-critical */ }
+  }
+
+  log.dim(`  → ${companionPath}`);
+  return companionName;
+}
+
+/**
+ * Recursively build a children object for a parent entity.
+ * Mutates parentObj to set parentObj.children = { column: [], join: [], filter: [] }.
+ * Returns companionFiles: string[] of written companion file basenames.
+ *
+ * Each child object retains _entity set to the physical entity name
+ * (output_value, output_value_entity_column_rel, output_value_filter)
+ * so that push can route submissions correctly.
+ *
+ * @param {Object} parentObj - The entity object to populate (mutated in place)
+ * @param {Object} node - Tree node from buildOutputHierarchyTree (has _children)
+ * @param {string} rootStem - Root output file stem (e.g. "_output~Sales~abc")
+ * @param {string} outputDir - Directory where root output JSON lives
+ * @param {string} serverTz - Server timezone
+ * @param {string} [parentStem] - Ancestor stem for compound companion naming
+ * @returns {Promise<string[]>} - Array of written companion file basenames
+ */
+async function buildInlineOutputChildren(parentObj, node, rootStem, outputDir, serverTz, parentStem = rootStem) {
+  const companionFiles = [];
+  const nodeChildren = node._children || {};
+
+  // Always create children object with all three doc keys
+  parentObj.children = { column: [], join: [], filter: [] };
+
+  for (const docKey of INLINE_DOC_KEYS) {
+    const entityArray = nodeChildren[docKey];
+    const physicalKey = INLINE_DOC_TO_PHYSICAL[docKey];
+
+    if (!Array.isArray(entityArray) || entityArray.length === 0) continue;
+
+    for (const child of entityArray) {
+      // Build a clean copy without tree-internal fields
+      const childObj = { ...child };
+      delete childObj._children;
+
+      // Decode any base64 values
+      for (const [key, value] of Object.entries(childObj)) {
+        if (key === 'CustomSQL') continue; // handled by extractCustomSqlIfNeeded
+        if (value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64') {
+          childObj[key] = resolveContentValue(value);
+        }
+      }
+
+      // Ensure _entity is set to physical entity name (for push routing)
+      childObj._entity = physicalKey;
+
+      // Compute companion stem for this child
+      const childStem = getChildCompanionStem(rootStem, physicalKey, child.UID, parentStem);
+
+      // Extract CustomSQL if needed
+      const companionFile = await extractCustomSqlIfNeeded(childObj, childStem, outputDir, serverTz);
+      if (companionFile) companionFiles.push(companionFile);
+
+      // Recurse into child's _children (e.g. join→column, column→filter)
+      if (child._children && Object.keys(child._children).some(k => child._children[k]?.length > 0)) {
+        const gcFiles = await buildInlineOutputChildren(childObj, child, rootStem, outputDir, serverTz, childStem);
+        companionFiles.push(...gcFiles);
+      } else {
+        // Leaf node: still set empty children
+        childObj.children = { column: [], join: [], filter: [] };
+      }
+
+      parentObj.children[docKey].push(childObj);
+    }
+  }
+
+  return companionFiles;
+}
+
+/**
+ * Move orphaned old-format child output .json files to /trash.
+ * Old format: _output~name~uid.column~name~uid.json (has .column~, .join~, or .filter~ segments)
+ *
+ * @param {string} outputDir - Directory containing output files
+ * @param {string} rootStem - Root output file stem (e.g. "_output~Sales~abc")
+ */
+async function trashOrphanedChildFiles(outputDir, rootStem) {
+  let files;
+  try { files = await readdir(outputDir); } catch { return; }
+
+  const trashDir = join(process.cwd(), 'trash');
+  let trashCreated = false;
+
+  for (const f of files) {
+    if (f.startsWith(`${rootStem}.`) && f.endsWith('.json') && !f.includes('.CustomSQL.')) {
+      // Check it's actually an old child file (has .column~, .join~, or .filter~ segment)
+      if (/\.(column|join|filter)~/.test(f)) {
+        if (!trashCreated) {
+          await mkdir(trashDir, { recursive: true });
+          trashCreated = true;
+        }
+        try {
+          await rename(join(outputDir, f), join(trashDir, f));
+          log.dim(`  Trashed orphaned child file: ${f}`);
+        } catch { /* non-critical */ }
+      }
+    }
+  }
+
+  // Re-apply trash icon if files were moved (self-heals after user clears trash)
+  if (trashCreated) {
+    await ensureTrashIcon(trashDir);
+  }
+}
+
+// ─── Filename Parsing ──────────────────────────────────────────────────────
+
 /**
  * Parse an output hierarchy filename back into entity relationships.
  *
@@ -2882,7 +3231,11 @@ export function parseOutputHierarchyFile(filename) {
 
 /**
  * Main orchestrator: process output hierarchy entities during clone.
- * Builds tree, resolves filenames, writes hierarchy files.
+ * Builds tree, resolves filenames, writes single-file inline format.
+ *
+ * Each root output produces exactly one .json file with all children
+ * embedded inline under children: { column: [], join: [], filter: [] }.
+ * Companion .sql files are extracted per CustomSQL rules.
  *
  * @param {Object} appJson - The full app JSON
  * @param {Object} structure - Bin hierarchy structure
@@ -2946,8 +3299,29 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
     const rootBasename = buildOutputFilename('output', output, filenameCols.output);
     const rootMetaPath = join(binDir, `${rootBasename}.json`);
 
+    // Detect old-format files that need migration to inline children format.
+    // Old format: children.column/join/filter contain @reference strings to separate files.
+    // New format: children contain inline entity objects directly.
+    let needsFormatMigration = false;
+    if (await fileExists(rootMetaPath)) {
+      try {
+        const existingMeta = JSON.parse(await readFile(rootMetaPath, 'utf8'));
+        if (existingMeta.children) {
+          const allRefs = [
+            ...(existingMeta.children.column || []),
+            ...(existingMeta.children.join || []),
+            ...(existingMeta.children.filter || []),
+          ];
+          needsFormatMigration = allRefs.some(ref => typeof ref === 'string' && ref.startsWith('@'));
+        }
+      } catch { /* read error — will be overwritten */ }
+      if (needsFormatMigration) {
+        log.info(`  Migrating ${rootBasename} to inline children format...`);
+      }
+    }
+
     // Change detection for existing files (skip when --entity forces re-processing)
-    if (await fileExists(rootMetaPath) && !options.yes && !forceReprocess) {
+    if (await fileExists(rootMetaPath) && !options.yes && !forceReprocess && !needsFormatMigration) {
       if (bulkAction.value === 'skip_all') {
         log.dim(`  Skipped ${rootBasename}`);
         refs.push({ uid: output.UID, metaPath: rootMetaPath });
@@ -2980,94 +3354,11 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
       }
     }
 
-    // Collect child file references for root JSON
-    const childRefs = { column: [], join: [], filter: [] };
-
-    // Helper to build a child filename segment
-    const childSegment = (type, node, col) => {
-      const uid = node.UID || '';
-      const rawName = node[col];
-      const name = rawName ? sanitizeFilename(String(rawName)) : '';
-      return (!name || name === uid) ? `${type}~${uid}` : `${type}~${name}~${uid}`;
-    };
-
-    // Process all children depth-first
-    // Direct filters on output
-    for (const filter of output._children.filter) {
-      const filterName = `${rootBasename}.${childSegment('filter', filter, filenameCols.output_value_filter)}`;
-      const filterPath = join(binDir, `${filterName}.json`);
-      await writeOutputEntityFile(filter, 'output_value_filter', filterPath, serverTz);
-      childRefs.filter.push(`@${filterPath}`);
-    }
-
-    // Direct columns on output
-    for (const col of output._children.column) {
-      const colSeg = childSegment('column', col, filenameCols.output_value);
-      const colName = `${rootBasename}.${colSeg}`;
-      const colPath = join(binDir, `${colName}.json`);
-      await writeOutputEntityFile(col, 'output_value', colPath, serverTz);
-      childRefs.column.push(`@${colPath}`);
-
-      // Filters under this column
-      for (const filter of col._children.filter) {
-        const filterName = `${colName}.${childSegment('filter', filter, filenameCols.output_value_filter)}`;
-        const filterPath = join(binDir, `${filterName}.json`);
-        await writeOutputEntityFile(filter, 'output_value_filter', filterPath, serverTz);
-        childRefs.filter.push(`@${filterPath}`);
-      }
-    }
-
-    // Joins on output
-    for (const j of output._children.join) {
-      const joinSeg = childSegment('join', j, filenameCols.output_value_entity_column_rel);
-      const joinName = `${rootBasename}.${joinSeg}`;
-      const joinPath = join(binDir, `${joinName}.json`);
-      await writeOutputEntityFile(j, 'output_value_entity_column_rel', joinPath, serverTz);
-      childRefs.join.push(`@${joinPath}`);
-
-      // Columns under this join
-      for (const col of j._children.column) {
-        const joinColName = `${joinName}.${childSegment('column', col, filenameCols.output_value)}`;
-        const joinColPath = join(binDir, `${joinColName}.json`);
-        await writeOutputEntityFile(col, 'output_value', joinColPath, serverTz);
-        childRefs.column.push(`@${joinColPath}`);
-
-        // Filters under this join→column
-        for (const filter of col._children.filter) {
-          const filterName = `${joinColName}.${childSegment('filter', filter, filenameCols.output_value_filter)}`;
-          const filterPath = join(binDir, `${filterName}.json`);
-          await writeOutputEntityFile(filter, 'output_value_filter', filterPath, serverTz);
-          childRefs.filter.push(`@${filterPath}`);
-        }
-      }
-    }
-
-    // Write root output JSON with child references
+    // Build clean root object (strip tree-internal fields)
     const rootMeta = {};
-    const rootContentColumns = [];
     for (const [key, value] of Object.entries(output)) {
       if (key === '_children') continue;
-
-      // Extract CustomSQL to companion .sql file when Type is CustomSQL (even if empty)
-      // or when the column has actual content
-      if (key === 'CustomSQL') {
-        const decoded = resolveContentValue(value);
-        const hasContent = decoded && decoded.trim();
-        if (output.Type === 'CustomSQL' || hasContent) {
-          const sqlFilePath = rootMetaPath.replace(/\.json$/, '.CustomSQL.sql');
-          await writeFile(sqlFilePath, hasContent ? decoded : '');
-          rootMeta[key] = `@${basename(sqlFilePath)}`;
-          rootContentColumns.push('CustomSQL');
-          if (serverTz && (output._CreatedOn || output._LastUpdated)) {
-            try { await setFileTimestamps(sqlFilePath, output._CreatedOn, output._LastUpdated, serverTz); } catch { /* non-critical */ }
-          }
-          log.dim(`  → ${sqlFilePath}`);
-          continue;
-        }
-        // Not CustomSQL type and empty — store inline
-        rootMeta[key] = '';
-        continue;
-      }
+      if (key === 'CustomSQL') continue; // handled by extractCustomSqlIfNeeded below
 
       if (value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64') {
         rootMeta[key] = resolveContentValue(value);
@@ -3076,10 +3367,16 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
       }
     }
     rootMeta._entity = 'output';
-    if (rootContentColumns.length > 0) {
-      rootMeta._contentColumns = rootContentColumns;
-    }
-    rootMeta.children = childRefs;
+
+    // Copy raw CustomSQL for extraction helper
+    rootMeta.CustomSQL = output.CustomSQL;
+
+    // Extract CustomSQL on root (rules 1/2/3)
+    await extractCustomSqlIfNeeded(rootMeta, rootBasename, binDir, serverTz);
+
+    // Embed all children under rootMeta.children = { column, join, filter }
+    await buildInlineOutputChildren(rootMeta, output, rootBasename, binDir, serverTz);
+    // rootMeta now has .children = { column: [...], join: [...], filter: [...] }
 
     // If user chose a bin for a BinID-less output, store it and mark as modified
     if (chosenBinId) {
@@ -3090,6 +3387,9 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
     await writeFile(rootMetaPath, JSON.stringify(rootMeta, null, 2) + '\n');
     log.success(`Saved ${rootMetaPath}`);
 
+    // Move orphaned old-format child .json files to /trash
+    await trashOrphanedChildFiles(binDir, rootBasename);
+
     // Set file timestamps to server's _LastUpdated so diff detection works.
     // Skip when chosenBinId is set — keep mtime at "now" so push detects the local edit.
     if (!chosenBinId && serverTz && (output._CreatedOn || output._LastUpdated)) {
@@ -3105,61 +3405,79 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
 }
 
 /**
- * Write a single output hierarchy entity file (column, join, or filter).
- * Handles CustomSQL extraction to companion .sql files.
+ * Write manifest.json to project root.
+ * If a manifest content record was cloned from the server, use its Content value.
+ * Otherwise, generate from appJson values (empty strings for missing fields).
  */
-async function writeOutputEntityFile(node, physicalEntity, filePath, serverTz) {
-  const meta = {};
-  const contentColumns = [];
-
-  for (const [key, value] of Object.entries(node)) {
-    if (key === '_children') continue;
-
-    // Extract CustomSQL to companion .sql file when Type is CustomSQL (even if empty)
-    // or when the column has actual content
-    if (key === 'CustomSQL') {
-      const decoded = resolveContentValue(value);
-      const hasContent = decoded && decoded.trim();
-      if (node.Type === 'CustomSQL' || hasContent) {
-        const sqlFilePath = filePath.replace(/\.json$/, '.CustomSQL.sql');
-        await writeFile(sqlFilePath, hasContent ? decoded : '');
-        meta[key] = `@${basename(sqlFilePath)}`;
-        contentColumns.push('CustomSQL');
-
-        if (serverTz && (node._CreatedOn || node._LastUpdated)) {
-          try {
-            await setFileTimestamps(sqlFilePath, node._CreatedOn, node._LastUpdated, serverTz);
-          } catch { /* non-critical */ }
+async function writeManifestJson(appJson, contentRefs) {
+  // 1. Search contentRefs for a manifest content record
+  for (const ref of contentRefs) {
+    let meta;
+    try {
+      meta = JSON.parse(await readFile(ref.metaPath, 'utf8'));
+    } catch { continue; }
+
+    const name = (meta.Name || '').toLowerCase();
+    const ext = (meta.Extension || '').toLowerCase();
+    if (name.startsWith('manifest') && ext === 'json') {
+      // Found manifest — read content file and write to project root
+      const contentRef = meta.Content;
+      if (contentRef && String(contentRef).startsWith('@')) {
+        const refFile = String(contentRef).substring(1);
+        const contentPath = refFile.startsWith('/')
+          ? join(process.cwd(), refFile)
+          : join(dirname(ref.metaPath), refFile);
+        try {
+          const content = await readFile(contentPath, 'utf8');
+          await writeFile('manifest.json', content);
+          log.dim('  manifest.json written to project root (from server content)');
+        } catch (err) {
+          log.warn(`  Could not write manifest.json from server content: ${err.message}`);
         }
-        log.dim(`  → ${sqlFilePath}`);
-        continue;
       }
-      // Not CustomSQL type and empty — store inline
-      meta[key] = '';
-      continue;
-    }
-
-    // Decode other base64 columns inline
-    if (value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64') {
-      meta[key] = resolveContentValue(value);
-    } else {
-      meta[key] = value;
+      return;
     }
   }
 
-  meta._entity = physicalEntity;
-  if (contentColumns.length > 0) {
-    meta._contentColumns = contentColumns;
+  // 2. No manifest content record — generate from appJson values
+  const shortName = appJson.ShortName || '';
+  const appName = appJson.Name || '';
+  const description = appJson.Description || '';
+
+  // Find background_color from extension children (widget descriptor matching ShortName)
+  let bgColor = '#ffffff';
+  if (shortName) {
+    const extensions = appJson.children?.extension || [];
+    for (const ext of extensions) {
+      const descriptor = resolveContentValue(ext.Descriptor) || ext.Descriptor;
+      const string1 = resolveContentValue(ext.String1) || ext.String1;
+      if (descriptor === 'widget' && string1 === shortName) {
+        bgColor = resolveContentValue(ext.String4) || ext.String4 || bgColor;
+        break;
+      }
+    }
   }
 
-  await writeFile(filePath, JSON.stringify(meta, null, 2) + '\n');
-  log.dim(`  → ${filePath}`);
+  const manifest = {
+    name: appName,
+    short_name: shortName,
+    description,
+    orientation: 'portrait',
+    start_url: shortName ? `/app/${shortName}/ui/` : '',
+    lang: 'en',
+    scope: shortName ? `/app/${shortName}/ui/` : '',
+    display_override: ['window-control-overlay', 'minimal-ui'],
+    display: 'standalone',
+    background_color: bgColor,
+    theme_color: '#000000',
+    id: shortName,
+    screenshots: [],
+    ios: {},
+    icons: [],
+  };
 
-  if (serverTz && (node._CreatedOn || node._LastUpdated)) {
-    try {
-      await setFileTimestamps(filePath, node._CreatedOn, node._LastUpdated, serverTz);
-    } catch { /* non-critical */ }
-  }
+  await writeFile('manifest.json', JSON.stringify(manifest, null, 2) + '\n');
+  log.dim('  manifest.json generated at project root (from app.json values)');
 }
 
 /**
@@ -3205,7 +3523,7 @@ async function saveAppJson(appJson, contentRefs, otherRefs, domain) {
 }
 
 /**
- * Save .app.json baseline file with decoded base64 values.
+ * Save .dbo/.app_baseline.json baseline file with decoded base64 values.
  * This file tracks the server state for delta detection.
  */
 async function saveBaselineFile(appJson) {
@@ -3218,14 +3536,14 @@ async function saveBaselineFile(appJson) {
   // Save to .app.json
   await saveAppJsonBaseline(baseline);
 
-  log.dim('  .app.json baseline created (system-managed, do not edit)');
+  log.dim('  .dbo/.app_baseline.json baseline created (system-managed, do not edit)');
 }
 
 /**
  * Recursively decode base64 fields in an object or array.
  * Modifies the input object in-place.
  */
-function decodeBase64Fields(obj) {
+export function decodeBase64Fields(obj) {
   if (!obj || typeof obj !== 'object') {
     return;
   }