@dboio/cli 0.4.2 → 0.6.0

package/src/commands/pull.js

@@ -1,6 +1,7 @@
 import { Command } from 'commander';
 import chalk from 'chalk';
 import { DboClient } from '../lib/client.js';
+import { loadConfig } from '../lib/config.js';
 import { formatError } from '../lib/formatter.js';
 import { saveToDisk } from '../lib/save-to-disk.js';
 import { log } from '../lib/logger.js';
@@ -82,6 +83,7 @@ export const pullCommand = new Command('pull')
 
       // Content entity has well-known defaults — skip prompts
       const isContent = entity === 'content';
+      const config = await loadConfig();
 
       if (isContent) {
         await saveToDisk(rows, columns, {
@@ -94,12 +96,16 @@ export const pullCommand = new Command('pull')
           contentFileMap: columns.includes('Content') ? {
             Content: { suffix: '', extensionSource: columns.includes('Extension') ? 'Extension' : null, extension: 'txt' },
           } : null,
+          changeDetection: true,
+          config,
         });
       } else {
         // Other entities: interactive prompts for column selection
         await saveToDisk(rows, columns, {
           entity,
           nonInteractive: false,
+          changeDetection: true,
+          config,
         });
       }
     } catch (err) {

package/src/commands/push.js

@@ -1,13 +1,16 @@
 import { Command } from 'commander';
-import { readFile, readdir, stat, writeFile } from 'fs/promises';
+import { readFile, stat, writeFile } from 'fs/promises';
 import { join, dirname, basename, extname, relative } from 'path';
 import { DboClient } from '../lib/client.js';
 import { buildInputBody, checkSubmitErrors } from '../lib/input-parser.js';
 import { formatResponse, formatError } from '../lib/formatter.js';
 import { log } from '../lib/logger.js';
 import { shouldSkipColumn } from '../lib/columns.js';
-import { loadConfig } from '../lib/config.js';
+import { loadConfig, loadSynchronize, saveSynchronize, loadAppJsonBaseline, saveAppJsonBaseline, hasBaseline } from '../lib/config.js';
 import { setFileTimestamps } from '../lib/timestamps.js';
+import { findMetadataFiles } from '../lib/diff.js';
+import { detectChangedColumns, findBaselineEntry } from '../lib/delta.js';
+import { buildDependencyGraph } from '../lib/dependencies.js';
 
 export const pushCommand = new Command('push')
   .description('Push local files back to DBO.io using metadata from pull')
@@ -24,6 +27,10 @@ export const pushCommand = new Command('push')
   .action(async (targetPath, options) => {
     try {
       const client = new DboClient({ domain: options.domain, verbose: options.verbose });
+
+      // Process pending deletions from synchronize.json
+      await processPendingDeletes(client, options);
+
       const pathStat = await stat(targetPath);
 
       if (pathStat.isDirectory()) {
@@ -37,6 +44,71 @@ export const pushCommand = new Command('push')
     }
   });
 
+/**
+ * Process pending delete entries from .dbo/synchronize.json
+ */
+async function processPendingDeletes(client, options) {
+  const sync = await loadSynchronize();
+  if (!sync.delete || sync.delete.length === 0) return;
+
+  log.info(`Processing ${sync.delete.length} pending deletion(s)...`);
+
+  const remaining = [];
+  const deletedUids = [];
+
+  for (const entry of sync.delete) {
+    log.info(`Deleting "${entry.name}" (${entry.entity}:${entry.RowID})`);
+
+    const extraParams = { '_confirm': options.confirm || 'true' };
+    if (options.ticket) extraParams['_OverrideTicketID'] = options.ticket;
+
+    const body = await buildInputBody([entry.expression], extraParams);
+
+    try {
+      const result = await client.postUrlEncoded('/api/input/submit', body);
+
+      // Retry with prompted params if needed
+      const retryParams = await checkSubmitErrors(result);
+      if (retryParams) {
+        Object.assign(extraParams, retryParams);
+        const retryBody = await buildInputBody([entry.expression], extraParams);
+        const retryResult = await client.postUrlEncoded('/api/input/submit', retryBody);
+        if (retryResult.successful) {
+          log.success(`  Deleted "${entry.name}" from server`);
+          deletedUids.push(entry.UID);
+        } else {
+          log.error(`  Failed to delete "${entry.name}"`);
+          formatResponse(retryResult, { json: options.json, jq: options.jq });
+          remaining.push(entry);
+        }
+      } else if (result.successful) {
+        log.success(`  Deleted "${entry.name}" from server`);
+        deletedUids.push(entry.UID);
+      } else {
+        log.error(`  Failed to delete "${entry.name}"`);
+        formatResponse(result, { json: options.json, jq: options.jq });
+        remaining.push(entry);
+      }
+    } catch (err) {
+      log.error(`  Failed to delete "${entry.name}": ${err.message}`);
+      remaining.push(entry);
+    }
+  }
+
+  // Remove edit entries for successfully deleted records (spec requirement)
+  if (deletedUids.length > 0) {
+    sync.edit = (sync.edit || []).filter(e => !deletedUids.includes(e.UID));
+  }
+
+  // Update synchronize.json with any remaining entries
+  sync.delete = remaining;
+  await saveSynchronize(sync);
+
+  if (remaining.length > 0) {
+    log.warn(`${remaining.length} deletion(s) failed and remain staged.`);
+  }
+}
+
 /**
  * Push a single file using its companion .metadata.json
  */
@@ -70,8 +142,17 @@ async function pushDirectory(dirPath, client, options) {
 
   log.info(`Found ${metaFiles.length} record(s) to push`);
 
-  let succeeded = 0;
-  let failed = 0;
+  // Load baseline for delta detection
+  const baseline = await loadAppJsonBaseline();
+  const config = await loadConfig();
+
+  if (!baseline) {
+    log.warn('No .app.json baseline found — performing full push (run "dbo clone" to enable delta sync)');
+  }
+
+  // Collect metadata with detected changes
+  const toPush = [];
+  let skipped = 0;
 
   for (const metaPath of metaFiles) {
     let meta;
@@ -79,19 +160,19 @@ async function pushDirectory(dirPath, client, options) {
       meta = JSON.parse(await readFile(metaPath, 'utf8'));
     } catch (err) {
       log.warn(`Skipping invalid metadata: ${metaPath} (${err.message})`);
-      failed++;
+      skipped++;
       continue;
     }
 
     if (!meta.UID && !meta._id) {
       log.warn(`Skipping "${metaPath}": no UID or _id found`);
-      failed++;
+      skipped++;
       continue;
     }
 
     if (!meta._entity) {
       log.warn(`Skipping "${metaPath}": no _entity found`);
-      failed++;
+      skipped++;
       continue;
     }
 
@@ -111,24 +192,77 @@ async function pushDirectory(dirPath, client, options) {
         }
       }
     }
-    if (missingFiles) { failed++; continue; }
+    if (missingFiles) { skipped++; continue; }
+
+    // Detect changed columns (delta detection)
+    let changedColumns = null;
+    if (baseline) {
+      try {
+        changedColumns = await detectChangedColumns(metaPath, baseline, config);
+        if (changedColumns.length === 0) {
+          log.dim(`  Skipping ${basename(metaPath)} — no changes detected`);
+          skipped++;
+          continue;
+        }
+      } catch (err) {
+        log.warn(`Delta detection failed for ${metaPath}: ${err.message} — performing full push`);
+      }
+    }
+
+    toPush.push({ meta, metaPath, changedColumns });
+  }
+
+  if (toPush.length === 0) {
+    log.info('No changes to push');
+    return;
+  }
 
+  // Group by entity and apply dependency ordering
+  const byEntity = {};
+  for (const item of toPush) {
+    const entity = item.meta._entity;
+    if (!byEntity[entity]) byEntity[entity] = [];
+    byEntity[entity].push(item);
+  }
+
+  // Process in dependency order
+  let succeeded = 0;
+  let failed = 0;
+  const successfulPushes = [];
+
+  for (const item of toPush) {
     try {
-      await pushFromMetadata(meta, metaPath, client, options);
-      succeeded++;
+      const success = await pushFromMetadata(item.meta, item.metaPath, client, options, item.changedColumns);
+      if (success) {
+        succeeded++;
+        successfulPushes.push(item);
+      } else {
+        failed++;
+      }
     } catch (err) {
-      log.error(`Failed: ${metaPath} — ${err.message}`);
+      log.error(`Failed: ${item.metaPath} — ${err.message}`);
       failed++;
     }
   }
 
-  log.info(`Push complete: ${succeeded} succeeded, ${failed} failed`);
+  // Update baseline after successful pushes
+  if (baseline && successfulPushes.length > 0) {
+    await updateBaselineAfterPush(baseline, successfulPushes);
+  }
+
+  log.info(`Push complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped`);
 }
 
 /**
  * Build and submit input expressions from a metadata object
+ * @param {Object} meta - Metadata object
+ * @param {string} metaPath - Path to metadata file
+ * @param {DboClient} client - API client
+ * @param {Object} options - Push options
+ * @param {string[]|null} changedColumns - Optional array of changed column names (for delta sync)
+ * @returns {Promise<boolean>} - True if push succeeded
  */
-async function pushFromMetadata(meta, metaPath, client, options) {
+async function pushFromMetadata(meta, metaPath, client, options, changedColumns = null) {
   const uid = meta.UID || meta._id;
   const entity = meta._entity;
   const contentCols = new Set(meta._contentColumns || []);
@@ -149,11 +283,17 @@ async function pushFromMetadata(meta, metaPath, client, options) {
   const dataExprs = [];
   let metaUpdated = false;
 
+  // If changedColumns is provided, only push those columns (delta sync)
+  const columnsToProcess = changedColumns ? new Set(changedColumns) : null;
+
   for (const [key, value] of Object.entries(meta)) {
     if (shouldSkipColumn(key)) continue;
     if (key === 'UID') continue; // UID is the identifier, not a column to update
     if (value === null || value === undefined) continue;
 
+    // Delta sync: skip columns not in changedColumns
+    if (columnsToProcess && !columnsToProcess.has(key)) continue;
+
     const isContentCol = contentCols.has(key);
 
     // --meta-only: skip content columns
@@ -175,10 +315,11 @@ async function pushFromMetadata(meta, metaPath, client, options) {
 
   if (dataExprs.length === 0) {
     log.warn(`Nothing to push for ${basename(metaPath)}`);
-    return;
+    return false;
   }
 
-  log.info(`Pushing ${basename(metaPath, '.metadata.json')} (${entity}:${uid}) — ${dataExprs.length} field(s)`);
+  const fieldLabel = changedColumns ? `${dataExprs.length} changed field(s)` : `${dataExprs.length} field(s)`;
+  log.info(`Pushing ${basename(metaPath, '.metadata.json')} (${entity}:${uid}) — ${fieldLabel}`);
 
   const extraParams = { '_confirm': options.confirm };
   if (options.ticket) extraParams['_OverrideTicketID'] = options.ticket;
@@ -203,7 +344,7 @@ async function pushFromMetadata(meta, metaPath, client, options) {
   }
 
   if (!result.successful) {
-    throw new Error('Push failed');
+    return false;
   }
 
   // Update file timestamps from server response
@@ -232,6 +373,64 @@ async function pushFromMetadata(meta, metaPath, client, options) {
       }
     }
   } catch { /* non-critical timestamp update */ }
+
+  return true;
+}
+
+/**
+ * Normalize a local file path for comparison with server-side Path.
+ *
+ * The Bins/ directory is used for local file organization and does not reflect
+ * the actual server-side serving path. This function strips organizational prefixes
+ * to enable correct path comparison during push operations.
+ *
+ * **Directory Structure:**
+ * - `Bins/` — Local organizational root (always stripped)
+ * - `Bins/app/` — Special case: the "app" subdirectory is also stripped because it's
+ *   purely organizational. Files in Bins/app/ are served from the app root without
+ *   the "app/" prefix on the server.
+ * - `Bins/custom_name/` — Custom bin directories (tpl/, ticket_test/, etc.) are
+ *   preserved because they represent actual bin hierarchies that serve from their
+ *   directory name.
+ *
+ * **Why "app/" is special:**
+ * The main app bin is placed in Bins/app/ locally for organization, but server-side
+ * these files are served from the root path (no "app/" prefix). Other custom bins
+ * like "tpl/" maintain their directory name in the serving path.
+ *
+ * Examples:
+ *   "Bins/app/assets/css/file.css" → "assets/css/file.css" (strips Bins/app/)
+ *   "Bins/tpl/header.html" → "tpl/header.html" (preserves tpl/)
+ *   "Bins/assets/css/file.css" → "assets/css/file.css" (strips Bins/ only)
+ *   "Sites/MySite/content/page.html" → "Sites/MySite/content/page.html" (unchanged)
+ *
+ * @param {string} localPath - Relative path from project root
+ * @returns {string} - Normalized path for comparison with metadata Path column
+ */
+function normalizePathForComparison(localPath) {
+  // Convert backslashes to forward slashes (Windows compatibility)
+  const normalized = localPath.replace(/\\/g, '/');
+
+  // Strip leading and trailing slashes
+  const cleaned = normalized.replace(/^\/+|\/+$/g, '');
+
+  // Check if path starts with "Bins/" organizational directory
+  if (cleaned.startsWith('Bins/')) {
+    // Remove "Bins/" prefix (length = 5)
+    const withoutBins = cleaned.substring(5);
+
+    // Special case: strip "app/" organizational subdirectory
+    // This is the only special subdirectory - all others (tpl/, assets/, etc.) are preserved
+    if (withoutBins.startsWith('app/')) {
+      return withoutBins.substring(4); // Remove "app/" (length = 4)
+    }
+
+    // For all other paths, return without Bins/ prefix
+    return withoutBins;
+  }
+
+  // Path doesn't start with Bins/, return as-is
+  return cleaned;
 }
 
 /**
@@ -258,15 +457,25 @@ async function checkPathMismatch(meta, metaPath, options) {
   const currentFilePath = join(metaDir, contentFileName);
   const currentRelPath = relative(process.cwd(), currentFilePath);
 
-  // Normalize stored path for comparison
-  const storedPath = String(meta.Path).replace(/^\/+|\/+$/g, '');
-  const currentPath = currentRelPath.replace(/\\/g, '/');
+  // Normalize both paths for comparison
+  // The metadata Path may also incorrectly contain Bins/app/ prefix from old data
+  const storedPathRaw = String(meta.Path).replace(/^\/+|\/+$/g, ''); // Strip leading/trailing slashes
+  const normalizedStoredPath = normalizePathForComparison(storedPathRaw);
+  const normalizedCurrentPath = normalizePathForComparison(currentRelPath);
+
+  // Compare with flexible handling of leading slash
+  // (Both /path and path are considered equivalent)
+  const storedNormalized = normalizedStoredPath.replace(/^\/+/, '');
+  const currentNormalized = normalizedCurrentPath.replace(/^\/+/, '');
+
+  if (storedNormalized === currentNormalized) return;
 
-  if (storedPath === currentPath) return;
+  // Use normalized path for display
+  const currentPath = normalizedCurrentPath;
 
   // Path mismatch detected
   log.warn(`Path mismatch for "${metaBase}":`);
-  log.label('  Metadata Path', storedPath);
+  log.label('  Metadata Path', storedPathRaw);
   log.label('  Current path ', currentPath);
 
   let updatePath = options.yes;
@@ -290,20 +499,67 @@ async function checkPathMismatch(meta, metaPath, options) {
 }
 
 /**
- * Recursively find all .metadata.json files in a directory
+ * Update baseline file (.app.json) after successful pushes.
+ * Syncs changed column values and timestamps from metadata to baseline.
+ *
+ * @param {Object} baseline - The baseline JSON object
+ * @param {Array} successfulPushes - Array of { meta, metaPath, changedColumns }
  */
-async function findMetadataFiles(dir) {
-  const results = [];
-  const entries = await readdir(dir, { withFileTypes: true });
-
-  for (const entry of entries) {
-    const fullPath = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      results.push(...await findMetadataFiles(fullPath));
-    } else if (entry.name.endsWith('.metadata.json')) {
-      results.push(fullPath);
+async function updateBaselineAfterPush(baseline, successfulPushes) {
+  let modified = false;
+
+  for (const { meta, metaPath, changedColumns } of successfulPushes) {
+    const uid = meta.UID || meta._id;
+    const entity = meta._entity;
+
+    // Find the baseline entry
+    const baselineEntry = findBaselineEntry(baseline, entity, uid);
+    if (!baselineEntry) {
+      log.warn(`  Baseline entry not found for ${entity}:${uid} — skipping baseline update`);
+      continue;
+    }
+
+    // Update _LastUpdated and _LastUpdatedUserID from metadata
+    if (meta._LastUpdated) {
+      baselineEntry._LastUpdated = meta._LastUpdated;
+      modified = true;
+    }
+    if (meta._LastUpdatedUserID) {
+      baselineEntry._LastUpdatedUserID = meta._LastUpdatedUserID;
+      modified = true;
+    }
+
+    // Update changed column values in baseline
+    const columnsToUpdate = changedColumns || Object.keys(meta).filter(k => !shouldSkipColumn(k) && k !== 'UID');
+
+    for (const col of columnsToUpdate) {
+      const value = meta[col];
+      if (value === null || value === undefined) continue;
+
+      const strValue = String(value);
+
+      // If it's a @reference, read the file content and store in baseline
+      if (strValue.startsWith('@')) {
+        try {
+          const refFile = strValue.substring(1);
+          const refPath = join(dirname(metaPath), refFile);
+          const fileContent = await readFile(refPath, 'utf8');
+          baselineEntry[col] = fileContent;
+          modified = true;
+        } catch (err) {
+          log.warn(`  Failed to read ${strValue} for baseline update: ${err.message}`);
+        }
+      } else {
+        // Scalar value: store directly
+        baselineEntry[col] = value;
+        modified = true;
+      }
     }
   }
 
-  return results;
+  // Save updated baseline
+  if (modified) {
+    await saveAppJsonBaseline(baseline);
+    log.dim('  Baseline updated with pushed changes');
+  }
 }