@dboio/cli 0.9.8 → 0.11.1

package/src/commands/rm.js

@@ -6,6 +6,7 @@ import { formatError } from '../lib/formatter.js';
 import { addDeleteEntry, removeAppJsonReference } from '../lib/config.js';
 import { findMetadataFiles } from '../lib/diff.js';
 import { loadStructureFile, findBinByPath, findChildBins, BINS_DIR } from '../lib/structure.js';
+import { runPendingMigrations } from '../lib/migrations.js';
 
 export const rmCommand = new Command('rm')
   .description('Remove a file or directory locally and stage server deletions for the next dbo push')
@@ -13,8 +14,10 @@ export const rmCommand = new Command('rm')
   .option('-f, --force', 'Skip confirmation prompts')
   .option('--keep-local', 'Only stage server deletion, do not delete local files')
   .option('--hard', 'Immediately delete local files (no Trash; legacy behavior)')
+  .option('--no-migrate', 'Skip pending migrations for this invocation')
   .action(async (targetPath, options) => {
     try {
+      await runPendingMigrations(options);
       const pathStat = await stat(targetPath).catch(() => null);
       if (!pathStat) {
         log.error(`Path not found: "${targetPath}"`);
@@ -172,7 +175,7 @@ async function rmFile(filePath, options) {
     process.exit(1);
   }
   if (!rowId) {
-    log.error(`No row ID found in "${metaPath}". Cannot build delete expression.`);
+    log.error(`No row ID found in "${metaPath}". Cannot build delete expression.\n  Metadata needs _id or ${entity.charAt(0).toUpperCase() + entity.slice(1)}ID. Run "dbo pull" to populate.`);
     process.exit(1);
   }
 

package/src/commands/status.js

@@ -5,10 +5,12 @@ import { access } from 'fs/promises';
 import { join } from 'path';
 import { homedir } from 'os';
 import { log } from '../lib/logger.js';
+import { runPendingMigrations, countPendingMigrations } from '../lib/migrations.js';
 
 export const statusCommand = new Command('status')
   .description('Show current DBO CLI configuration and session status')
-  .action(async () => {
+  .option('--no-migrate', 'Skip pending migrations for this invocation')
+  .action(async (options) => {
     try {
       const initialized = await isInitialized();
       const config = await loadConfig();
@@ -40,6 +42,15 @@ export const statusCommand = new Command('status')
       const transactionKeyPreset = await loadTransactionKeyPreset();
       log.label('Transaction Key', transactionKeyPreset || '(not set — defaults to RowUID)');
 
+      // Run pending migrations (no-op if --no-migrate)
+      await runPendingMigrations(options);
+
+      // Report pending count (after running, so completed ones are excluded)
+      const pending = await countPendingMigrations();
+      if (pending > 0) {
+        log.label('Pending migrations', `${pending} (will run on next command; use --no-migrate to skip)`);
+      }
+
       // Display plugin status
       const scopes = await getAllPluginScopes();
       const pluginNames = Object.keys(scopes);

package/src/commands/sync.js

@@ -0,0 +1,71 @@
+import { Command } from 'commander';
+import { log } from '../lib/logger.js';
+import { loadConfig, loadAppConfig, saveAppJsonBaseline } from '../lib/config.js';
+import { DboClient } from '../lib/client.js';
+import { decodeBase64Fields } from './clone.js';
+import { runPendingMigrations } from '../lib/migrations.js';
+
+export const syncCommand = new Command('sync')
+  .description('Synchronise local state with the server')
+  .option('--baseline', 'Re-fetch server state and update .dbo/.app_baseline.json (does not modify local files)')
+  .option('--no-migrate', 'Skip pending migrations for this invocation')
+  .action(async (options) => {
+    await runPendingMigrations(options);
+    if (!options.baseline) {
+      log.warn('No sync mode specified. Use --baseline to reset the baseline file.');
+      process.exit(1);
+    }
+
+    const config = await loadConfig();
+    if (!config.domain) {
+      log.error('No domain configured. Run "dbo init" first.');
+      process.exit(1);
+    }
+
+    const appConfig = await loadAppConfig();
+    if (!appConfig.AppShortName) {
+      log.error('No AppShortName found. Run "dbo clone" first.');
+      process.exit(1);
+    }
+
+    const ora = (await import('ora')).default;
+    const spinner = ora('Syncing baseline from server...').start();
+
+    const client = new DboClient({ domain: config.domain, verbose: options.verbose });
+
+    let result;
+    try {
+      result = await client.get(`/api/app/object/${appConfig.AppShortName}`);
+    } catch (err) {
+      spinner.fail(`Failed to fetch app JSON: ${err.message}`);
+      process.exit(1);
+    }
+
+    const data = result.payload || result.data;
+
+    let appRecord;
+    if (Array.isArray(data)) {
+      appRecord = data.length > 0 ? data[0] : null;
+    } else if (data?.Rows?.length > 0) {
+      appRecord = data.Rows[0];
+    } else if (data?.rows?.length > 0) {
+      appRecord = data.rows[0];
+    } else if (data && typeof data === 'object' && (data.UID || data.ShortName)) {
+      appRecord = data;
+    } else {
+      appRecord = null;
+    }
+
+    if (!appRecord) {
+      spinner.fail(`No app found with ShortName "${appConfig.AppShortName}"`);
+      process.exit(1);
+    }
+
+    // Deep clone and decode base64 fields (same logic as clone's saveBaselineFile)
+    const baseline = JSON.parse(JSON.stringify(appRecord));
+    decodeBase64Fields(baseline);
+
+    await saveAppJsonBaseline(baseline);
+    spinner.succeed('.dbo/.app_baseline.json updated from server');
+    log.dim('  Run "dbo push" to sync local changes against the new baseline');
+  });

package/src/lib/client.js

@@ -123,6 +123,16 @@ export class DboClient {
     return this._parseResponse(response);
   }
 
+  /**
+   * Clear the server-side cache. Must be called after POST transactions so that
+   * subsequent GET requests (diff, pull, toe-stepping) return fresh data.
+   */
+  async voidCache() {
+    try {
+      await this.request('/?voidcache=true');
+    } catch { /* best-effort — don't block on failure */ }
+  }
+
   /**
    * Fetch a URL and return the raw response as a Buffer (for binary downloads).
    */

package/src/lib/config.js

@@ -1,4 +1,4 @@
-import { readFile, writeFile, mkdir, access } from 'fs/promises';
+import { readFile, writeFile, mkdir, access, chmod, unlink } from 'fs/promises';
 import { join } from 'path';
 import { log } from './logger.js';
 
@@ -8,7 +8,7 @@ const CONFIG_LOCAL_FILE = 'config.local.json';
 const CREDENTIALS_FILE = 'credentials.json';
 const COOKIES_FILE = 'cookies.txt';
 const SYNCHRONIZE_FILE = 'synchronize.json';
-const BASELINE_FILE = '.app.json';
+const BASELINE_FILE = '.app_baseline.json';
 
 function dboDir() {
   return join(process.cwd(), DBO_DIR);
@@ -525,6 +525,37 @@ export async function getAllPluginScopes() {
   return result;
 }
 
+// ─── Migration tracking (config.local.json._completedMigrations) ──────────
+
+/**
+ * Load the list of completed migration IDs from .dbo/config.local.json.
+ * Returns an empty array if the file does not exist or the key is absent.
+ * @returns {Promise<string[]>}
+ */
+export async function loadCompletedMigrations() {
+  try {
+    const local = await loadLocalConfig();
+    const ids = local._completedMigrations;
+    return Array.isArray(ids) ? ids : [];
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Append a migration ID to .dbo/config.local.json._completedMigrations.
+ * Deduplicates: if the ID is already present, no-op.
+ * @param {string} id - Three-digit migration ID, e.g. '001'
+ */
+export async function saveCompletedMigration(id) {
+  const local = await loadLocalConfig();
+  const existing = new Set(Array.isArray(local._completedMigrations) ? local._completedMigrations : []);
+  if (existing.has(id)) return; // already recorded — idempotent
+  existing.add(id);
+  local._completedMigrations = [...existing].sort();
+  await saveLocalConfig(local);
+}
+
 // ─── Output Hierarchy Filename Preferences ────────────────────────────────
 
 /**
@@ -693,10 +724,10 @@ export async function ensureGitignore(patterns) {
   for (const p of toAdd) log.dim(`  Added ${p} to .gitignore`);
 }
 
-// ─── Baseline (.app.json) ─────────────────────────────────────────────────
+// ─── Baseline (.dbo/.app_baseline.json) ───────────────────────────────────
 
-function baselinePath() {
-  return join(process.cwd(), BASELINE_FILE);
+export function baselinePath() {
+  return join(dboDir(), BASELINE_FILE);
 }
 
 /**
@@ -707,11 +738,38 @@ export async function hasBaseline() {
 }
 
 /**
- * Load .app.json baseline file (tracks server state for delta detection).
+ * Load .dbo/.app_baseline.json baseline file.
+ * Auto-migrates from legacy root .app.json if the new path does not exist.
  */
 export async function loadAppJsonBaseline() {
+  const newPath = baselinePath();
+
+  // Legacy migration: root .app.json → .dbo/.app_baseline.json
+  if (!(await exists(newPath))) {
+    const legacyPath = join(process.cwd(), '.app.json');
+    if (await exists(legacyPath)) {
+      let parsed;
+      try {
+        parsed = JSON.parse(await readFile(legacyPath, 'utf8'));
+      } catch {
+        log.warn('Could not migrate .app.json — file is not valid JSON. Delete it manually and run "dbo clone" to recreate the baseline.');
+        return null;
+      }
+      await mkdir(dboDir(), { recursive: true });
+      await writeFile(newPath, JSON.stringify(parsed, null, 2) + '\n');
+      try { await chmod(newPath, 0o444); } catch { /* ignore */ }
+      try {
+        await unlink(legacyPath);
+      } catch {
+        log.warn('Migrated baseline but could not delete root .app.json — please remove it manually.');
+      }
+      log.dim('Migrated .app.json → .dbo/.app_baseline.json (system-managed baseline)');
+      return parsed;
+    }
+  }
+
   try {
-    const raw = await readFile(baselinePath(), 'utf8');
+    const raw = await readFile(newPath, 'utf8');
     return JSON.parse(raw);
   } catch {
     return null;
@@ -719,10 +777,24 @@ export async function loadAppJsonBaseline() {
 }
 
 /**
- * Save .app.json baseline file.
+ * Save .dbo/.app_baseline.json baseline file.
+ * Temporarily widens permissions before writing (chmod 0o644),
+ * then restores read-only (chmod 0o444) after writing.
  */
 export async function saveAppJsonBaseline(data) {
+  await mkdir(dboDir(), { recursive: true });
+
+  try {
+    await chmod(baselinePath(), 0o644);
+  } catch { /* file doesn't exist yet — first write */ }
+
   await writeFile(baselinePath(), JSON.stringify(data, null, 2) + '\n');
+
+  try {
+    await chmod(baselinePath(), 0o444);
+  } catch {
+    log.warn('⚠ Could not set baseline file permissions — ensure .dbo/.app_baseline.json is not manually edited');
+  }
 }
 
 /**

package/src/lib/delta.js

@@ -1,37 +1,26 @@
 import { readFile, stat } from 'fs/promises';
 import { join, dirname } from 'path';
 import { log } from './logger.js';
+import { loadAppJsonBaseline, saveAppJsonBaseline } from './config.js';
 
 /**
- * Load the baseline file (.app.json) from disk.
+ * Load the baseline file from disk.
+ * Delegates to loadAppJsonBaseline() in config.js for the canonical path.
  *
- * @param {string} cwd - Current working directory
  * @returns {Promise<Object|null>} - Baseline JSON or null if not found
  */
-export async function loadBaseline(cwd = process.cwd()) {
-  const baselinePath = join(cwd, '.app.json');
-  try {
-    const raw = await readFile(baselinePath, 'utf8');
-    return JSON.parse(raw);
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      return null; // Baseline doesn't exist
-    }
-    log.warn(`Failed to parse .app.json: ${err.message}`);
-    return null;
-  }
+export async function loadBaseline() {
+  return loadAppJsonBaseline();
 }
 
 /**
- * Save baseline data to .app.json file.
+ * Save baseline data to disk.
+ * Delegates to saveAppJsonBaseline() in config.js for the canonical path.
  *
  * @param {Object} data - Baseline JSON data
- * @param {string} cwd - Current working directory
  */
-export async function saveBaseline(data, cwd = process.cwd()) {
-  const { writeFile } = await import('fs/promises');
-  const baselinePath = join(cwd, '.app.json');
-  await writeFile(baselinePath, JSON.stringify(data, null, 2), 'utf8');
+export async function saveBaseline(data) {
+  return saveAppJsonBaseline(data);
 }
 
 /**
@@ -158,6 +147,17 @@ export async function detectChangedColumns(metaPath, baseline) {
     }
   }
 
+  // Check for columns present in baseline but removed from local metadata
+  // (user deleted the key or set it to null → should clear on server)
+  for (const [columnName, baselineVal] of Object.entries(baselineEntry)) {
+    if (shouldSkipColumn(columnName)) continue;
+    if (columnName in metadata) continue; // already compared above
+    // Baseline has a non-null value but local metadata doesn't have this key at all
+    if (baselineVal !== null && baselineVal !== undefined && normalizeValue(baselineVal) !== '') {
+      changedColumns.push(columnName);
+    }
+  }
+
   // Check _mediaFile for binary file changes (media entities)
   if (metadata._mediaFile && isReference(metadata._mediaFile)) {
     const mediaPath = resolveReferencePath(metadata._mediaFile, metaDir);
@@ -180,7 +180,7 @@ export async function detectChangedColumns(metaPath, baseline) {
  * @param {Object} metadata - Metadata object
  * @returns {string[]} - Array of user column names
  */
-function getAllUserColumns(metadata) {
+export function getAllUserColumns(metadata) {
   return Object.keys(metadata).filter(col => !shouldSkipColumn(col));
 }
 
@@ -190,7 +190,7 @@ function getAllUserColumns(metadata) {
  * @param {string} columnName - Column name
  * @returns {boolean} - True if should skip
  */
-function shouldSkipColumn(columnName) {
+export function shouldSkipColumn(columnName) {
   // Skip system columns starting with underscore, UID, and children
   return columnName.startsWith('_') ||
          columnName === 'UID' ||
@@ -203,7 +203,7 @@ function shouldSkipColumn(columnName) {
  * @param {*} value - Value to check
  * @returns {boolean} - True if reference
  */
-function isReference(value) {
+export function isReference(value) {
   return typeof value === 'string' && value.startsWith('@');
 }
 
@@ -215,7 +215,7 @@ function isReference(value) {
  * @param {string} baseDir - Base directory containing metadata
  * @returns {string} - Absolute file path
  */
-function resolveReferencePath(reference, baseDir) {
+export function resolveReferencePath(reference, baseDir) {
   const refPath = reference.substring(1); // Strip leading @
   if (refPath.startsWith('/')) {
     return join(process.cwd(), refPath);
@@ -229,7 +229,7 @@ function resolveReferencePath(reference, baseDir) {
  * @param {*} value - Value to normalize
  * @returns {string} - Normalized string value
  */
-function normalizeValue(value) {
+export function normalizeValue(value) {
   if (value === null || value === undefined) {
     return '';
   }
@@ -241,3 +241,156 @@ function normalizeValue(value) {
 
   return String(value).trim();
 }
+
+// ─── Compound Output Delta Detection ────────────────────────────────────────
+
+const _OUTPUT_DOC_KEYS = ['column', 'join', 'filter'];
+
+/**
+ * Detect changed columns across a compound output file (root + inline children).
+ *
+ * @param {string} metaPath - Path to the root output JSON file
+ * @param {Object} baseline - The baseline JSON
+ * @returns {Promise<{ root: string[], children: Object<string, string[]> }>}
+ *   root: changed column names on the root output entity
+ *   children: { uid: [changedCols] } for each inline child entity
+ */
+export async function detectOutputChanges(metaPath, baseline) {
+  const metaRaw = await readFile(metaPath, 'utf8');
+  const meta = JSON.parse(metaRaw);
+  const metaDir = dirname(metaPath);
+
+  // Detect root changes
+  const root = await _detectEntityChanges(meta, baseline, metaDir);
+
+  // Detect child changes recursively via children.{column,join,filter}
+  const children = {};
+  if (meta.children) {
+    await _walkChildrenForChanges(meta.children, baseline, metaDir, children);
+  }
+
+  return { root, children };
+}
+
+async function _detectEntityChanges(entity, baseline, metaDir) {
+  const { UID, _entity } = entity;
+  const baselineEntry = findBaselineEntry(baseline, _entity, UID);
+  if (!baselineEntry) {
+    return getAllUserColumns(entity);
+  }
+  const changed = [];
+  for (const [col, val] of Object.entries(entity)) {
+    if (shouldSkipColumn(col) || col === 'children') continue;
+    if (isReference(val)) {
+      const refPath = resolveReferencePath(val, metaDir);
+      if (await compareFileContent(refPath, baselineEntry[col])) changed.push(col);
+    } else {
+      if (normalizeValue(val) !== normalizeValue(baselineEntry[col])) {
+        // Skip Extension column when baseline is null/empty (same as detectChangedColumns)
+        if (col === 'Extension'
+            && !normalizeValue(baselineEntry[col])
+            && (baselineEntry[col] === null || baselineEntry[col] === undefined)) {
+          continue;
+        }
+        changed.push(col);
+      }
+    }
+  }
+  // Check for columns removed from entity but present in baseline
+  for (const [col, baselineVal] of Object.entries(baselineEntry)) {
+    if (shouldSkipColumn(col) || col === 'children') continue;
+    if (col in entity) continue;
+    if (baselineVal !== null && baselineVal !== undefined && normalizeValue(baselineVal) !== '') {
+      changed.push(col);
+    }
+  }
+  return changed;
+}
+
+// ─── Bin Delta Detection ─────────────────────────────────────────────────────
+
+/** Columns tracked for bin change detection */
+const BIN_TRACKED_COLUMNS = ['Name', 'Path', 'ParentBinID', 'Active', 'Public'];
+
+/**
+ * Detect changes between a current bin entry (from structure.json) and the
+ * baseline (from app.json children.bin array).
+ *
+ * @param {Object} binEntry - Current bin entry from structure.json (with binId, name, path, etc.)
+ * @param {Object} baseline - The baseline JSON (app.json)
+ * @returns {string[]} - Array of changed column names
+ */
+export function detectBinChanges(binEntry, baseline) {
+  if (!baseline || !baseline.children || !Array.isArray(baseline.children.bin)) {
+    return BIN_TRACKED_COLUMNS; // no baseline → treat all as changed
+  }
+
+  const baselineBin = baseline.children.bin.find(
+    b => b.UID === binEntry.uid || b.BinID === binEntry.binId
+  );
+
+  if (!baselineBin) {
+    return BIN_TRACKED_COLUMNS; // new bin, not in baseline
+  }
+
+  const changed = [];
+
+  // Map structure.json field names → server column names for comparison
+  const fieldMap = {
+    Name:        { current: binEntry.name,        baseline: baselineBin.Name },
+    Path:        { current: binEntry.path,        baseline: baselineBin.Path },
+    ParentBinID: { current: binEntry.parentBinID, baseline: baselineBin.ParentBinID },
+  };
+
+  for (const [col, { current, baseline: base }] of Object.entries(fieldMap)) {
+    const curStr = current !== null && current !== undefined ? String(current) : '';
+    const baseStr = base !== null && base !== undefined ? String(base) : '';
+    if (curStr !== baseStr) changed.push(col);
+  }
+
+  // Direct column comparisons for Active/Public (may exist on baseline)
+  for (const col of ['Active', 'Public']) {
+    if (baselineBin[col] !== undefined && binEntry[col] !== undefined) {
+      if (String(binEntry[col]) !== String(baselineBin[col])) {
+        changed.push(col);
+      }
+    }
+  }
+
+  return changed;
+}
+
+/**
+ * Synthesize a virtual metadata object for a bin entry (for use with pushFromMetadata).
+ *
+ * @param {Object} binEntry - Bin entry from structure.json
+ * @param {number} [appId] - AppID from app config
+ * @returns {Object} - Virtual metadata object
+ */
+export function synthesizeBinMetadata(binEntry, appId) {
+  const meta = {
+    _entity: 'bin',
+    _id: binEntry.binId,
+    UID: binEntry.uid,
+    Name: binEntry.name,
+    Path: binEntry.path,
+    ParentBinID: binEntry.parentBinID,
+  };
+  if (appId) meta.AppID = appId;
+  return meta;
+}
+
+async function _walkChildrenForChanges(childrenObj, baseline, metaDir, result) {
+  for (const docKey of _OUTPUT_DOC_KEYS) {
+    const entityArray = childrenObj[docKey];
+    if (!Array.isArray(entityArray) || entityArray.length === 0) continue;
+    for (const child of entityArray) {
+      const changed = await _detectEntityChanges(child, baseline, metaDir);
+      result[child.UID] = changed;
+      // Recurse into child's own children (e.g. column[0].children.filter)
+      if (child.children) {
+        await _walkChildrenForChanges(child.children, baseline, metaDir, result);
+      }
+    }
+  }
+}