@dboio/cli 0.4.2 → 0.5.1

package/src/commands/clone.js

@@ -2,24 +2,28 @@ import { Command } from 'commander';
 import { readFile, writeFile, mkdir, access } from 'fs/promises';
 import { join, basename, extname } from 'path';
 import { DboClient } from '../lib/client.js';
-import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore } from '../lib/config.js';
-import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, BINS_DIR, DEFAULT_PROJECT_DIRS } from '../lib/structure.js';
+import { loadConfig, updateConfigWithApp, loadClonePlacement, saveClonePlacement, ensureGitignore, saveEntityDirPreference, loadEntityDirPreference } from '../lib/config.js';
+import { buildBinHierarchy, resolveBinPath, createDirectories, saveStructureFile, getBinName, findBinByPath, BINS_DIR, DEFAULT_PROJECT_DIRS, ENTITY_DIR_MAP } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
 import { setFileTimestamps } from '../lib/timestamps.js';
+import { getLocalSyncTime, isServerNewer, hasLocalModifications, promptChangeDetection, inlineDiffAndMerge } from '../lib/diff.js';
 
 /**
  * Resolve a column value that may be base64-encoded.
  * Server returns large text as: { bytes: N, value: "base64string", encoding: "base64" }
  */
-function resolveContentValue(value) {
+export function resolveContentValue(value) {
   if (value && typeof value === 'object' && !Array.isArray(value)
-      && value.encoding === 'base64' && typeof value.value === 'string') {
-    return Buffer.from(value.value, 'base64').toString('utf8');
+      && value.encoding === 'base64') {
+    // Base64 object: decode if value is a string, otherwise empty string
+    return typeof value.value === 'string'
+      ? Buffer.from(value.value, 'base64').toString('utf8')
+      : '';
   }
   return value !== null && value !== undefined ? String(value) : null;
 }
 
-function sanitizeFilename(name) {
+export function sanitizeFilename(name) {
   return name.replace(/[/\\?%*:|"<>]/g, '-').replace(/\s+/g, '-').substring(0, 200);
 }
 
@@ -27,6 +31,45 @@ async function fileExists(path) {
   try { await access(path); return true; } catch { return false; }
 }
 
+/**
+ * Resolve a content Path to a directory under Bins/.
+ *
+ * When a record has BinID=null but a Path like "tpl/ticket-menu-list.html"
+ * or "app/nima-test-test.txt", we need to place it inside Bins/:
+ *
+ * 1. Extract the directory portion of the Path (e.g. "tpl", "app")
+ * 2. Try to match it to an existing bin in structure.json
+ *    - If found → use the resolved bin path (e.g. "Bins/app")
+ * 3. If not found → place under Bins/<dir> (e.g. "Bins/tpl")
+ *
+ * This ensures content files always land inside Bins/, never at the project root.
+ */
+function resolvePathToBinsDir(pathValue, structure) {
+  const cleaned = pathValue.replace(/^\/+|\/+$/g, '');
+  if (!cleaned) return BINS_DIR;
+
+  // Extract the directory portion (strip filename if path contains one)
+  const pathExt = extname(cleaned);
+  let dirPart;
+  if (pathExt) {
+    const lastSlash = cleaned.lastIndexOf('/');
+    dirPart = lastSlash >= 0 ? cleaned.substring(0, lastSlash) : null;
+  } else {
+    dirPart = cleaned;
+  }
+
+  if (!dirPart) return BINS_DIR;
+
+  // Try to match the directory path to an existing bin in structure.json
+  const bin = findBinByPath(dirPart, structure);
+  if (bin) {
+    return resolveBinPath(bin.binId, structure);
+  }
+
+  // No matching bin — place under Bins/<dir>
+  return `${BINS_DIR}/${dirPart}`;
+}
+
 export const cloneCommand = new Command('clone')
   .description('Clone an app from DBO.io to a local project structure')
   .argument('[source]', 'Local JSON file path (optional)')
@@ -163,9 +206,18 @@ export async function performClone(source, options = {}) {
     if (['bin', 'content', 'output', 'media'].includes(entityName)) continue;
     if (!Array.isArray(entries)) continue;
 
-    const refs = await processGenericEntries(entityName, entries, structure, options, placementPrefs.contentPlacement, serverTz);
-    if (refs.length > 0) {
-      otherRefs[entityName] = refs;
+    if (ENTITY_DIR_MAP[entityName]) {
+      // Entity types with project directories — process into their directory
+      const refs = await processEntityDirEntries(entityName, entries, options, serverTz);
+      if (refs.length > 0) {
+        otherRefs[entityName] = refs;
+      }
+    } else {
+      // Other entity types — process into Bins/ (requires BinID)
+      const refs = await processGenericEntries(entityName, entries, structure, options, placementPrefs.contentPlacement, serverTz);
+      if (refs.length > 0) {
+        otherRefs[entityName] = refs;
+      }
     }
   }
 
@@ -337,11 +389,12 @@ async function processContentEntries(contents, structure, options, contentPlacem
   const placementPreference = {
     value: (contentPlacement === 'path' || contentPlacement === 'bin') ? contentPlacement : null,
   };
+  const bulkAction = { value: null }; // Shared across records for overwrite_all / skip_all
 
   log.info(`Processing ${contents.length} content record(s)...`);
 
   for (const record of contents) {
-    const ref = await processRecord('content', record, structure, options, usedNames, placementPreference, serverTz);
+    const ref = await processRecord('content', record, structure, options, usedNames, placementPreference, serverTz, bulkAction);
     if (ref) refs.push(ref);
   }
 
@@ -360,12 +413,13 @@ async function processGenericEntries(entityName, entries, structure, options, co
   const placementPreference = {
     value: (contentPlacement === 'path' || contentPlacement === 'bin') ? contentPlacement : null,
   };
+  const bulkAction = { value: null };
   let processed = 0;
 
   for (const record of entries) {
     if (!record.BinID) continue; // Skip entries without BinID
 
-    const ref = await processRecord(entityName, record, structure, options, usedNames, placementPreference, serverTz);
+    const ref = await processRecord(entityName, record, structure, options, usedNames, placementPreference, serverTz, bulkAction);
     if (ref) {
       refs.push(ref);
       processed++;
@@ -379,6 +433,270 @@ async function processGenericEntries(entityName, entries, structure, options, co
   return refs;
 }
 
+/**
+ * Process entity-dir entries: entities that map to DEFAULT_PROJECT_DIRS.
+ * These don't require a BinID — they go directly into their project directory
+ * (e.g., Extensions/, Data Sources/, etc.)
+ *
+ * Returns array of { uid, metaPath } for app.json reference replacement.
+ */
+async function processEntityDirEntries(entityName, entries, options, serverTz) {
+  if (!entries || entries.length === 0) return [];
+
+  const dirName = ENTITY_DIR_MAP[entityName];
+  if (!dirName) return [];
+
+  await mkdir(dirName, { recursive: true });
+
+  const refs = [];
+  const usedNames = new Map();
+  const bulkAction = { value: null };
+  const config = await loadConfig();
+
+  // Determine filename column: saved preference, or prompt, or default
+  let filenameCol = null;
+
+  if (options.yes) {
+    // Non-interactive: use Name, fallback UID
+    filenameCol = null; // will resolve per-record below
+  } else {
+    // Check saved preference
+    const savedCol = await loadEntityDirPreference(entityName);
+    if (savedCol) {
+      filenameCol = savedCol;
+      log.dim(`  Using saved filename column "${filenameCol}" for ${entityName}`);
+    } else {
+      // Prompt user to pick a filename column from available columns
+      const sampleRecord = entries[0];
+      const columns = Object.keys(sampleRecord).filter(k => k !== 'children' && !k.startsWith('_'));
+
+      if (columns.length > 0) {
+        const inquirer = (await import('inquirer')).default;
+
+        // Find best default
+        const defaultCol = columns.includes('Name') ? 'Name'
+          : columns.includes('name') ? 'name'
+          : columns.includes('UID') ? 'UID'
+          : columns[0];
+
+        const { col } = await inquirer.prompt([{
+          type: 'list',
+          name: 'col',
+          message: `Which column should be used as the filename for ${entityName} records?`,
+          choices: columns,
+          default: defaultCol,
+        }]);
+        filenameCol = col;
+
+        // Save preference for future clones
+        await saveEntityDirPreference(entityName, filenameCol);
+        log.dim(`  Saved filename column preference for ${entityName}`);
+      }
+    }
+  }
+
+  // Detect base64 content columns and prompt per-column for extraction
+  let contentColsToExtract = [];
+  if (!options.yes) {
+    // Collect base64 columns with a content snippet from the first record that has data
+    const base64Cols = []; // { col, snippet }
+    for (const record of entries) {
+      for (const [key, value] of Object.entries(record)) {
+        if (key === 'children' || key.startsWith('_')) continue;
+        if (value && typeof value === 'object' && !Array.isArray(value)
+            && value.encoding === 'base64' && value.value !== null) {
+          if (!base64Cols.find(c => c.col === key)) {
+            // Decode a snippet for preview
+            let snippet = '';
+            try {
+              const decoded = Buffer.from(value.value, 'base64').toString('utf8');
+              snippet = decoded.substring(0, 80).replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
+              if (decoded.length > 80) snippet += '...';
+            } catch { /* ignore decode errors */ }
+            base64Cols.push({ col: key, snippet });
+          }
+        }
+      }
+    }
+
+    if (base64Cols.length > 0) {
+      const inquirer = (await import('inquirer')).default;
+
+      // Prompt per column: show snippet and ask extract yes/no + extension
+      for (const { col, snippet } of base64Cols) {
+        const preview = snippet ? ` (${snippet})` : '';
+        const guessed = guessExtensionForColumn(col);
+
+        const { extract } = await inquirer.prompt([{
+          type: 'confirm',
+          name: 'extract',
+          message: `Extract "${col}" as companion file?${preview}`,
+          default: true,
+        }]);
+
+        if (extract) {
+          const { ext } = await inquirer.prompt([{
+            type: 'input',
+            name: 'ext',
+            message: `File extension for "${col}":`,
+            default: guessed,
+          }]);
+          contentColsToExtract.push({ col, ext: ext.replace(/^\./, '') });
+        }
+      }
+    }
+  }
+
+  log.info(`Processing ${entries.length} ${entityName} record(s) → ${dirName}/`);
+
+  for (const record of entries) {
+    // Determine filename for this record
+    let name;
+    if (filenameCol && record[filenameCol] !== null && record[filenameCol] !== undefined) {
+      name = sanitizeFilename(String(record[filenameCol]));
+    } else if (record.Name) {
+      name = sanitizeFilename(String(record.Name));
+    } else {
+      name = sanitizeFilename(String(record.UID || 'untitled'));
+    }
+
+    // Deduplicate filenames
+    const nameKey = `${dirName}/${name}`;
+    const count = usedNames.get(nameKey) || 0;
+    usedNames.set(nameKey, count + 1);
+    const finalName = count > 0 ? `${name}-${count + 1}` : name;
+
+    const metaPath = join(dirName, `${finalName}.metadata.json`);
+
+    // Change detection for existing files
+    // Skip change detection when user has selected new content columns to extract —
+    // we need to re-process all records to create the companion files
+    const hasNewExtractions = contentColsToExtract.length > 0;
+    if (await fileExists(metaPath) && !options.yes && !hasNewExtractions) {
+      if (bulkAction.value === 'skip_all') {
+        log.dim(`  Skipped ${finalName}`);
+        refs.push({ uid: record.UID, metaPath });
+        continue;
+      }
+
+      if (bulkAction.value !== 'overwrite_all') {
+        const localSyncTime = await getLocalSyncTime(metaPath);
+        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+
+        if (serverNewer) {
+          const action = await promptChangeDetection(finalName, record, config);
+
+          if (action === 'skip') {
+            log.dim(`  Skipped ${finalName}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+          if (action === 'skip_all') {
+            bulkAction.value = 'skip_all';
+            log.dim(`  Skipped ${finalName}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+          if (action === 'overwrite_all') {
+            bulkAction.value = 'overwrite_all';
+          }
+          if (action === 'compare') {
+            await inlineDiffAndMerge(record, metaPath, config);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+        } else {
+          const locallyModified = await hasLocalModifications(metaPath, config);
+          if (locallyModified) {
+            const action = await promptChangeDetection(finalName, record, config, { localIsNewer: true });
+
+            if (action === 'skip') {
+              log.dim(`  Kept local: ${finalName}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'skip_all') {
+              bulkAction.value = 'skip_all';
+              log.dim(`  Kept local: ${finalName}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'overwrite_all') {
+              bulkAction.value = 'overwrite_all';
+            }
+            if (action === 'compare') {
+              await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+          } else {
+            log.dim(`  Up to date: ${finalName}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+        }
+      }
+    }
+
+    // Build metadata
+    const meta = {};
+    const extractedContentCols = [];
+
+    for (const [key, value] of Object.entries(record)) {
+      if (key === 'children') continue;
+
+      // Check if this column should be extracted as a companion file
+      const extractInfo = contentColsToExtract.find(c => c.col === key);
+      if (extractInfo && value && typeof value === 'object' && value.encoding === 'base64' && value.value !== null) {
+        const decoded = resolveContentValue(value);
+        if (decoded) {
+          const colFileName = `${finalName}.${key}.${extractInfo.ext}`;
+          const colFilePath = join(dirName, colFileName);
+          await writeFile(colFilePath, decoded);
+          meta[key] = `@${colFileName}`;
+          extractedContentCols.push(key);
+
+          // Set timestamps on companion file
+          if (serverTz && (record._CreatedOn || record._LastUpdated)) {
+            try {
+              await setFileTimestamps(colFilePath, record._CreatedOn, record._LastUpdated, serverTz);
+            } catch { /* non-critical */ }
+          }
+
+          log.dim(`  → ${colFilePath}`);
+          continue;
+        }
+      }
+
+      // Other base64 columns not selected for extraction — decode inline
+      if (value && typeof value === 'object' && !Array.isArray(value) && value.encoding === 'base64') {
+        meta[key] = resolveContentValue(value);
+      } else {
+        meta[key] = value;
+      }
+    }
+
+    meta._entity = entityName;
+    if (extractedContentCols.length > 0) {
+      meta._contentColumns = extractedContentCols;
+    }
+
+    await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+    log.success(`Saved ${metaPath}`);
+
+    // Set file timestamps from server dates
+    if (serverTz && (record._CreatedOn || record._LastUpdated)) {
+      try {
+        await setFileTimestamps(metaPath, record._CreatedOn, record._LastUpdated, serverTz);
+      } catch { /* non-critical */ }
+    }
+
+    refs.push({ uid: record.UID, metaPath });
+  }
+
+  return refs;
+}
+
 /**
  * Process media entries: download binary files from server + create metadata.
  * Media uses Filename (not Name) and files are fetched via /api/media/{uid}.
@@ -424,6 +742,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
   const placementPreference = {
     value: mediaPlacement === 'fullpath' ? 'path' : mediaPlacement === 'bin' ? 'bin' : null,
   };
+  const mediaBulkAction = { value: null };
   let downloaded = 0;
   let failed = 0;
 
@@ -503,6 +822,76 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
     const filePath = join(dir, finalFilename);
     const metaPath = join(dir, `${finalFilename}.metadata.json`);
 
+    // Change detection for existing media files
+    if (await fileExists(metaPath) && !options.yes) {
+      if (mediaBulkAction.value === 'skip_all') {
+        log.dim(`  Skipped ${finalFilename}`);
+        refs.push({ uid: record.UID, metaPath });
+        continue;
+      }
+
+      if (mediaBulkAction.value !== 'overwrite_all') {
+        const localSyncTime = await getLocalSyncTime(metaPath);
+        const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+
+        if (serverNewer) {
+          const action = await promptChangeDetection(dedupName, record, config);
+
+          if (action === 'skip') {
+            log.dim(`  Skipped ${finalFilename}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+          if (action === 'skip_all') {
+            mediaBulkAction.value = 'skip_all';
+            log.dim(`  Skipped ${finalFilename}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+          if (action === 'overwrite_all') {
+            mediaBulkAction.value = 'overwrite_all';
+          }
+          if (action === 'compare') {
+            // For binary media, show metadata diffs only
+            await inlineDiffAndMerge(record, metaPath, config);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+        } else {
+          // Server _LastUpdated hasn't changed — check for local modifications
+          const locallyModified = await hasLocalModifications(metaPath, config);
+          if (locallyModified) {
+            const action = await promptChangeDetection(dedupName, record, config, { localIsNewer: true });
+
+            if (action === 'skip') {
+              log.dim(`  Kept local: ${finalFilename}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'skip_all') {
+              mediaBulkAction.value = 'skip_all';
+              log.dim(`  Kept local: ${finalFilename}`);
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            if (action === 'overwrite_all') {
+              mediaBulkAction.value = 'overwrite_all';
+            }
+            if (action === 'compare') {
+              await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+              refs.push({ uid: record.UID, metaPath });
+              continue;
+            }
+            // 'overwrite' falls through to download
+          } else {
+            log.dim(`  Up to date: ${finalFilename}`);
+            refs.push({ uid: record.UID, metaPath });
+            continue;
+          }
+        }
+      }
+    }
+
     // Download the file
     try {
       const buffer = await client.getBuffer(`/api/media/${record.UID}`);
@@ -551,7 +940,7 @@ async function processMediaEntries(mediaRecords, structure, options, config, app
  * Process a single record: determine directory, write content file + metadata.
  * Returns { uid, metaPath } or null.
  */
-async function processRecord(entityName, record, structure, options, usedNames, placementPreference, serverTz) {
+async function processRecord(entityName, record, structure, options, usedNames, placementPreference, serverTz, bulkAction = { value: null }) {
   let name = sanitizeFilename(String(record.Name || record.UID || 'untitled'));
   const ext = (record.Extension || 'txt').toLowerCase();
 
@@ -603,7 +992,8 @@ async function processRecord(entityName, record, structure, options, usedNames,
   } else if (hasBinID) {
     dir = resolveBinPath(record.BinID, structure);
   } else if (hasPath) {
-    dir = record.Path;
+    // BinID is null — resolve Path into a Bins/ location
+    dir = resolvePathToBinsDir(record.Path, structure);
   }
 
   // Clean up directory path
@@ -636,6 +1026,71 @@ async function processRecord(entityName, record, structure, options, usedNames,
   const filePath = join(dir, fileName);
   const metaPath = join(dir, `${finalName}.metadata.json`);
 
+  // Change detection: check if file already exists locally
+  if (await fileExists(metaPath) && !options.yes) {
+    if (bulkAction.value === 'skip_all') {
+      log.dim(`  Skipped ${finalName}.${ext}`);
+      return { uid: record.UID, metaPath };
+    }
+
+    if (bulkAction.value !== 'overwrite_all') {
+      const localSyncTime = await getLocalSyncTime(metaPath);
+      const config = await loadConfig();
+      const serverNewer = isServerNewer(localSyncTime, record._LastUpdated, config);
+
+      if (serverNewer) {
+        const action = await promptChangeDetection(finalName, record, config);
+
+        if (action === 'skip') {
+          log.dim(`  Skipped ${finalName}.${ext}`);
+          return { uid: record.UID, metaPath };
+        }
+        if (action === 'skip_all') {
+          bulkAction.value = 'skip_all';
+          log.dim(`  Skipped ${finalName}.${ext}`);
+          return { uid: record.UID, metaPath };
+        }
+        if (action === 'overwrite_all') {
+          bulkAction.value = 'overwrite_all';
+          // Fall through to write
+        }
+        if (action === 'compare') {
+          await inlineDiffAndMerge(record, metaPath, config);
+          return { uid: record.UID, metaPath };
+        }
+        // 'overwrite' falls through to normal write
+      } else {
+        // Server _LastUpdated hasn't changed since last sync.
+        // Check if local content files were modified (user edits).
+        const locallyModified = await hasLocalModifications(metaPath, config);
+        if (locallyModified) {
+          const action = await promptChangeDetection(finalName, record, config, { localIsNewer: true });
+
+          if (action === 'skip') {
+            log.dim(`  Kept local: ${finalName}.${ext}`);
+            return { uid: record.UID, metaPath };
+          }
+          if (action === 'skip_all') {
+            bulkAction.value = 'skip_all';
+            log.dim(`  Kept local: ${finalName}.${ext}`);
+            return { uid: record.UID, metaPath };
+          }
+          if (action === 'overwrite_all') {
+            bulkAction.value = 'overwrite_all';
+          }
+          if (action === 'compare') {
+            await inlineDiffAndMerge(record, metaPath, config, { localIsNewer: true });
+            return { uid: record.UID, metaPath };
+          }
+          // 'overwrite' falls through to normal write
+        } else {
+          log.dim(`  Up to date: ${finalName}.${ext}`);
+          return { uid: record.UID, metaPath };
+        }
+      }
+    }
+  }
+
   if (hasContent) {
     const decoded = resolveContentValue(contentValue);
     if (decoded) {
@@ -694,7 +1149,7 @@ async function processRecord(entityName, record, structure, options, usedNames,
 /**
  * Guess file extension for a column name.
  */
-function guessExtensionForColumn(columnName) {
+export function guessExtensionForColumn(columnName) {
   const lower = columnName.toLowerCase();
   if (lower.includes('css')) return 'css';
   if (lower.includes('js') || lower.includes('script')) return 'js';