@dboio/cli 0.11.2 → 0.11.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dboio/cli",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "description": "CLI for the DBO.io framework",
   "type": "module",
   "bin": {

package/src/commands/clone.js

@@ -2956,8 +2956,8 @@ async function trashOrphanedChildFiles(outputDir, rootStem) {
         log.dim(`  Trashed orphaned child file: ${f}`);
       } catch { /* non-critical */ }
     }
-    // Also trash the legacy root file itself (_output~Name~UID.json) if new format exists
-    if (matchesLegacy === false && f === `${legacyStem}.json`) {
+    // Also trash the legacy root file itself (_output~Name~UID.json or .metadata.json) if new format exists
+    if (matchesLegacy === false && (f === `${legacyStem}.json` || f === `${legacyStem}.metadata.json`)) {
       if (!trashCreated) {
         await mkdir(trashDir, { recursive: true });
         trashCreated = true;
@@ -2984,9 +2984,10 @@ async function trashOrphanedChildFiles(outputDir, rootStem) {
  * @returns {Object} - { segments: [{entity, name, uid}], rootOutputUid, entityType, uid }
  */
 export function parseOutputHierarchyFile(filename) {
-  // Strip .json extension
+  // Strip .metadata.json or legacy .json extension
   let base = filename;
-  if (base.endsWith('.json')) base = base.substring(0, base.length - 5);
+  if (base.endsWith('.metadata.json')) base = base.substring(0, base.length - 14);
+  else if (base.endsWith('.json')) base = base.substring(0, base.length - 5);
 
   // Split into segments by finding entity type boundaries
   // Entity types are: output~ (or legacy _output~), column~, join~, filter~
@@ -3109,7 +3110,14 @@ async function processOutputHierarchy(appJson, structure, options, serverTz) {
 
     // Build root output filename
     const rootBasename = buildOutputFilename('output', output, filenameCols.output);
-    const rootMetaPath = join(binDir, `${rootBasename}.json`);
+    let rootMetaPath = join(binDir, `${rootBasename}.metadata.json`);
+
+    // Legacy fallback: if old .json exists but new .metadata.json doesn't, rename in-place
+    const legacyJsonPath = join(binDir, `${rootBasename}.json`);
+    if (!await fileExists(rootMetaPath) && await fileExists(legacyJsonPath)) {
+      await rename(legacyJsonPath, rootMetaPath);
+      log.dim(`  Renamed ${rootBasename}.json → ${rootBasename}.metadata.json`);
+    }
 
     // Detect old-format files that need migration to inline children format.
     // Old format: children.column/join/filter contain @reference strings to separate files.

package/src/commands/push.js

@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import { readFile, readdir, stat, writeFile, rename as fsRename, mkdir, access } from 'fs/promises';
+import { readFile, stat, writeFile, rename as fsRename, mkdir, access } from 'fs/promises';
 import { join, dirname, basename, extname, relative } from 'path';
 import { DboClient } from '../lib/client.js';
 import { buildInputBody, checkSubmitErrors, getSessionUserOverride } from '../lib/input-parser.js';
@@ -306,18 +306,17 @@ async function pushSingleFile(filePath, client, options, modifyKey = null, trans
     const baseline = await loadAppJsonBaseline();
     if (baseline) {
       const appConfig = await loadAppConfig();
-      const proceed = await checkToeStepping([{ meta, metaPath }], client, baseline, options, appConfig?.AppShortName);
-      if (!proceed) return;
+      const result = await checkToeStepping([{ meta, metaPath }], client, baseline, options, appConfig?.AppShortName);
+      if (result === false || result instanceof Set) return;
     }
   }
 
   await pushFromMetadata(meta, metaPath, client, options, null, modifyKey, transactionKey);
 }
-
 /**
- * Ensure manifest.json at project root has companion metadata in bins/app/.
- * If manifest.json exists but no manifest*.metadata.json is in bins/app/,
- * auto-create the metadata so the push flow picks it up.
+ * Ensure manifest.json at project root has companion metadata in lib/bins/app/.
+ * Only creates metadata if manifest.json exists at root AND no existing metadata
+ * anywhere in the project already references @/manifest.json (prevents duplicates).
  */
 async function ensureManifestMetadata() {
   // Check if manifest.json exists at project root
@@ -327,30 +326,24 @@ async function ensureManifestMetadata() {
     return; // No manifest.json — nothing to do
   }
 
-  // Check if bins/app/ already has metadata that references @/manifest.json.
-  // A filename-only check (startsWith('manifest')) is insufficient because
-  // the metadata may have been renamed with a ~UID suffix or prefixed with
-  // __WILL_DELETE__.  Instead, scan actual metadata content for the reference.
-  const binsAppDir = join(process.cwd(), 'bins', 'app');
-  try {
-    const entries = await readdir(binsAppDir);
-    const metaEntries = entries.filter(e => e.endsWith('.metadata.json'));
-    for (const entry of metaEntries) {
-      try {
-        const raw = await readFile(join(binsAppDir, entry), 'utf8');
-        const parsed = JSON.parse(raw);
-        if (parsed.Content === '@/manifest.json') return; // Already tracked
-      } catch { /* skip unreadable files */ }
-    }
-  } catch {
-    // bins/app/ doesn't exist — will create it
+  // Scan the entire project for any metadata file that already references manifest.json.
+  // This prevents creating duplicates when the metadata lives in an unexpected location.
+  const ig = await loadIgnore();
+  const allMeta = await findMetadataFiles(process.cwd(), ig);
+  for (const metaPath of allMeta) {
+    try {
+      const raw = await readFile(metaPath, 'utf8');
+      const parsed = JSON.parse(raw);
+      if (parsed.Content === '@/manifest.json') return; // Already tracked
+    } catch { /* skip unreadable */ }
   }
 
-  // Auto-create manifest.metadata.json
+  // No existing metadata references manifest.json — create one
   const appConfig = await loadAppConfig();
   const structure = await loadStructureFile();
   const appBin = findBinByPath('app', structure);
 
+  const binsAppDir = join(process.cwd(), BINS_DIR, 'app');
   await mkdir(binsAppDir, { recursive: true });
 
   const meta = {
@@ -481,8 +474,15 @@ async function pushDirectory(dirPath, client, options, modifyKey = null, transac
     const toCheck = toPush.filter(item => !item.isNew);
     if (toCheck.length > 0) {
       const appConfig = await loadAppConfig();
-      const proceed = await checkToeStepping(toCheck, client, baseline, options, appConfig?.AppShortName);
-      if (!proceed) return;
+      const result = await checkToeStepping(toCheck, client, baseline, options, appConfig?.AppShortName);
+      if (result === false) return; // user cancelled entirely
+      if (result instanceof Set) {
+        // Filter out skipped UIDs
+        for (let i = toPush.length - 1; i >= 0; i--) {
+          if (result.has(toPush[i].meta.UID)) toPush.splice(i, 1);
+        }
+        if (toPush.length === 0) return;
+      }
     }
   }
 
@@ -725,8 +725,15 @@ async function pushByUIDs(uids, client, options, modifyKey = null, transactionKe
     }
     if (toCheck.length > 0) {
       const appConfig = await loadAppConfig();
-      const proceed = await checkToeStepping(toCheck, client, baseline, options, appConfig?.AppShortName);
-      if (!proceed) return;
+      const result = await checkToeStepping(toCheck, client, baseline, options, appConfig?.AppShortName);
+      if (result === false) return; // user cancelled entirely
+      if (result instanceof Set) {
+        // Filter out skipped UIDs from matches
+        for (let i = matches.length - 1; i >= 0; i--) {
+          if (result.has(matches[i].meta?.UID)) matches.splice(i, 1);
+        }
+        if (matches.length === 0) return;
+      }
     }
   }
 

package/src/lib/client.js

@@ -124,11 +124,13 @@ export class DboClient {
   }
 
   /**
-   * Clear the server-side cache. Must be called after POST transactions so that
-   * subsequent GET requests (diff, pull, toe-stepping) return fresh data.
+   * Clear the server-side cache so that subsequent GET requests return fresh data.
+   * Called before comparison fetches (toe-stepping, diff) and after POST submissions.
    */
   async voidCache() {
     try {
+      const baseUrl = await this.getBaseUrl();
+      if (this.verbose) log.verbose(`VoidCache: ${baseUrl}/?voidcache=true`);
       await this.request('/?voidcache=true');
     } catch { /* best-effort — don't block on failure */ }
   }

package/src/lib/toe-stepping.js

@@ -1,7 +1,9 @@
-import { dirname, basename } from 'path';
+import chalk from 'chalk';
+import { dirname, basename, join } from 'path';
 import { readFile } from 'fs/promises';
 import { findBaselineEntry, shouldSkipColumn, normalizeValue, isReference, resolveReferencePath } from './delta.js';
 import { resolveContentValue } from '../commands/clone.js';
+import { computeLineDiff, formatDiff } from './diff.js';
 import { log } from './logger.js';
 
 /**
@@ -274,7 +276,8 @@ function findOldestBaselineDate(records, baseline) {
  * @param {Object} baseline - Loaded baseline from .dbo/.app_baseline.json
  * @param {Object} options  - Commander options (options.yes used for auto-accept)
  * @param {string} [appShortName] - App short name for bulk fetch (optional)
- * @returns {Promise<boolean>} - true = proceed, false = user cancelled
+ * @returns {Promise<boolean|Set<string>>} - true = proceed with all,
+ *   false = user cancelled entirely, Set<string> = UIDs to skip (proceed with rest)
  */
 export async function checkToeStepping(records, client, baseline, options, appShortName) {
   // Build list of records to check (skip new records without UID)
@@ -302,6 +305,8 @@ export async function checkToeStepping(records, client, baseline, options, appSh
 
   // Fall back to per-record fetches if batch returned nothing or wasn't available
   if (!serverRecords || serverRecords.size === 0) {
+    // Void cache if batch path was skipped (batch already calls voidCache internally)
+    if (!appShortName) await client.voidCache();
     spinner.text = `Fetching ${requests.length} record(s) from server...`;
     serverRecords = await fetchServerRecords(client, requests);
   }
@@ -314,7 +319,11 @@ export async function checkToeStepping(records, client, baseline, options, appSh
 
   spinner.succeed(`Fetched ${serverRecords.size} record(s) from server`);
 
-  const conflicts = [];
+  // Detect conflicts and prompt per-record
+  const inquirer = (await import('inquirer')).default;
+  let skippedUIDs = new Set();
+  let bulkAction = null; // 'push_all' | 'skip_all'
+  let hasConflicts = false;
 
   for (const { meta, metaPath } of records) {
     const uid = meta.UID;
@@ -339,6 +348,8 @@ export async function checkToeStepping(records, client, baseline, options, appSh
     if (isNaN(serverDate) || isNaN(baselineDate)) continue; // unparseable — skip
     if (serverDate <= baselineDate) continue; // server is same or older — no conflict
 
+    hasConflicts = true;
+
     // Conflict detected: server changed since our baseline
     const metaDir = dirname(metaPath);
     const label = basename(metaPath, '.metadata.json');
@@ -346,36 +357,123 @@ export async function checkToeStepping(records, client, baseline, options, appSh
     const serverUser = serverEntry._LastUpdatedUserID || 'unknown';
 
     displayConflict(label, serverUser, serverTs, diffColumns);
-    conflicts.push({ label, serverUser });
-  }
 
-  if (conflicts.length === 0) return true; // no conflicts
+    // Auto-accept or auto-skip based on bulk action
+    if (options.yes || bulkAction === 'push_all') {
+      continue; // push this record
+    }
+    if (bulkAction === 'skip_all') {
+      skippedUIDs.add(uid);
+      continue;
+    }
 
-  log.warn('');
-  log.warn(`  ${conflicts.length} record(s) have server changes that would be overwritten.`);
-  log.warn('');
+    // Per-record prompt with compare option
+    let action;
+    while (true) {
+      ({ action } = await inquirer.prompt([{
+        type: 'list',
+        name: 'action',
+        message: `"${label}" has server changes. How to proceed?`,
+        choices: [
+          { name: 'Push anyway (overwrite server changes)', value: 'push' },
+          { name: 'Compare differences', value: 'compare' },
+          { name: 'Skip this record', value: 'skip' },
+          { name: 'Push all remaining (overwrite all)', value: 'push_all' },
+          { name: 'Skip all remaining', value: 'skip_all' },
+          { name: 'Cancel entire push', value: 'cancel' },
+        ],
+      }]));
+
+      if (action === 'compare') {
+        await showPushDiff(serverEntry, meta, metaPath);
+        continue; // re-prompt after showing diff
+      }
+      break;
+    }
 
-  if (options.yes) {
-    log.dim('  --yes flag: proceeding despite server conflicts');
-    return true;
+    if (action === 'cancel') {
+      log.info('Push cancelled. Run "dbo pull" to fetch server changes first.');
+      return false;
+    }
+    if (action === 'skip' || action === 'skip_all') {
+      skippedUIDs.add(uid);
+      if (action === 'skip_all') bulkAction = 'skip_all';
+    }
+    if (action === 'push_all') {
+      bulkAction = 'push_all';
+    }
   }
 
-  const inquirer = (await import('inquirer')).default;
-  const { action } = await inquirer.prompt([{
-    type: 'list',
-    name: 'action',
-    message: 'Server has newer changes. How would you like to proceed?',
-    choices: [
-      { name: 'Overwrite server changes (push anyway)', value: 'overwrite' },
-      { name: 'Cancel — pull server changes first', value: 'cancel' },
-    ],
-  }]);
-
-  if (action === 'cancel') {
-    log.info('Push cancelled. Run "dbo pull" to fetch server changes first.');
-    return false;
+  if (!hasConflicts) return true;
+
+  // Return skipped UIDs so the caller can filter them out
+  if (skippedUIDs.size > 0) {
+    log.dim(`  Skipping ${skippedUIDs.size} conflicting record(s)`);
+    return skippedUIDs;
   }
 
-  log.dim('  Proceeding — local changes will overwrite server state.');
   return true;
 }
+
+/**
+ * Show a read-only diff between the server state and local metadata/content
+ * for a single record during push toe-stepping.
+ *
+ * Green (+) = local (what you're about to push)
+ * Red (-)   = server (what would be overwritten)
+ */
+async function showPushDiff(serverEntry, localMeta, metaPath) {
+  const metaDir = dirname(metaPath);
+  const contentCols = localMeta._contentColumns || [];
+
+  // Compare content file columns
+  for (const col of contentCols) {
+    const localRef = localMeta[col];
+    if (!localRef || !String(localRef).startsWith('@')) continue;
+
+    const localFilePath = join(metaDir, String(localRef).substring(1));
+    let localContent = '';
+    try {
+      localContent = await readFile(localFilePath, 'utf8');
+    } catch {
+      localContent = '';
+    }
+
+    const serverValue = resolveContentValue(serverEntry[col]);
+    if (serverValue === null) continue;
+
+    if (localContent !== serverValue) {
+      // Green = local (what we're pushing), Red = server (what gets overwritten)
+      const diff = computeLineDiff(serverValue, localContent);
+      log.plain('');
+      log.label('Field', `${col} (content file)`);
+      const formatted = formatDiff(diff, {
+        localLabel: `server: ${col}`,
+        serverLabel: `local: ${col}`,
+      });
+      for (const line of formatted) log.plain(line);
+    }
+  }
+
+  // Compare metadata fields
+  const skipFields = new Set(['_entity', '_contentColumns', '_mediaFile', 'children', '_pathConfirmed']);
+  for (const col of Object.keys(serverEntry)) {
+    if (skipFields.has(col)) continue;
+    if (contentCols.includes(col)) continue;
+    if (shouldSkipColumn(col)) continue;
+
+    const serverVal = serverEntry[col];
+    const localVal = localMeta[col];
+    const serverStr = serverVal != null ? String(resolveContentValue(serverVal) ?? serverVal) : '';
+    const localStr = localVal != null ? String(localVal) : '';
+
+    if (serverStr !== localStr) {
+      log.plain('');
+      log.label('Field', col);
+      if (serverStr) log.plain(chalk.red(`  - server: ${serverStr.substring(0, 200)}`));
+      if (localStr) log.plain(chalk.green(`  + local:  ${localStr.substring(0, 200)}`));
+    }
+  }
+
+  log.plain('');
+}

package/src/migrations/005-rename-output-metadata.js

@@ -0,0 +1,91 @@
+import { readdir, rename, access } from 'fs/promises';
+import { join } from 'path';
+import { log } from '../lib/logger.js';
+
+export const description = 'Rename output .json files to .metadata.json for consistency';
+
+/**
+ * Migration 005 — Rename output root metadata files from .json to .metadata.json.
+ *
+ * Old:  Sales~abc123.json
+ * New:  Sales~abc123.metadata.json
+ *
+ * Only renames root output files (contain ~ and end with .json).
+ * Skips .CustomSQL.sql companions, old-format child files (.column~, .join~, .filter~),
+ * and files that already use .metadata.json.
+ */
+export default async function run(_options) {
+  const cwd = process.cwd();
+  let totalRenamed = 0;
+
+  const dirs = await findDirsWithOutputJsonFiles(cwd);
+
+  for (const dir of dirs) {
+    const entries = await readdir(dir);
+
+    for (const name of entries) {
+      // Must be a .json file with ~ (UID separator) that is NOT already .metadata.json
+      if (!name.endsWith('.json')) continue;
+      if (name.endsWith('.metadata.json')) continue;
+      if (!name.includes('~')) continue;
+
+      // Skip CustomSQL companions
+      if (name.includes('.CustomSQL.')) continue;
+
+      // Skip old-format child files (.column~, .join~, .filter~)
+      if (/\.(column|join|filter)~/.test(name)) continue;
+
+      const newName = name.replace(/\.json$/, '.metadata.json');
+      const oldPath = join(dir, name);
+      const newPath = join(dir, newName);
+
+      // Skip if destination already exists
+      try {
+        await access(newPath);
+        continue;
+      } catch { /* good — doesn't exist */ }
+
+      await rename(oldPath, newPath);
+      totalRenamed++;
+    }
+  }
+
+  if (totalRenamed > 0) {
+    log.dim(`  Renamed ${totalRenamed} output file${totalRenamed === 1 ? '' : 's'} (.json → .metadata.json)`);
+  }
+}
+
+/**
+ * Recursively find directories that contain output .json files (with ~ in name).
+ * Skips .dbo/, node_modules/, trash/, .git/.
+ */
+async function findDirsWithOutputJsonFiles(root) {
+  const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+  const results = [];
+
+  async function walk(dir) {
+    let entries;
+    try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; }
+
+    let hasOutputFiles = false;
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        if (!SKIP.has(entry.name)) {
+          await walk(join(dir, entry.name));
+        }
+      } else if (
+        entry.name.endsWith('.json') &&
+        !entry.name.endsWith('.metadata.json') &&
+        entry.name.includes('~') &&
+        !entry.name.includes('.CustomSQL.') &&
+        !/\.(column|join|filter)~/.test(entry.name)
+      ) {
+        hasOutputFiles = true;
+      }
+    }
+    if (hasOutputFiles) results.push(dir);
+  }
+
+  await walk(root);
+  return results;
+}