@dboio/cli 0.19.7 → 0.20.3

package/src/lib/toe-stepping.js

@@ -1,11 +1,11 @@
 import chalk from 'chalk';
 import { dirname, basename, join } from 'path';
-import { readFile } from 'fs/promises';
-import { findBaselineEntry, shouldSkipColumn, normalizeValue, isReference, resolveReferencePath } from './delta.js';
+import { readFile, writeFile } from 'fs/promises';
+import { findBaselineEntry, shouldSkipColumn, normalizeValue, isReference, resolveReferencePath, saveBaseline } from './delta.js';
 import { resolveContentValue } from '../commands/clone.js';
 import { computeLineDiff, formatDiff } from './diff.js';
 import { parseMetaFilename } from './filenames.js';
-import { parseServerDate } from './timestamps.js';
+import { parseServerDate, setFileTimestamps } from './timestamps.js';
 import { log } from './logger.js';
 
 /**
@@ -335,6 +335,7 @@ export async function checkToeStepping(records, client, baseline, options, appSh
   let skippedUIDs = new Set();
   let bulkAction = null; // 'push_all' | 'skip_all'
   let hasConflicts = false;
+  let baselineModified = false;
 
   for (const { meta, metaPath } of records) {
     const uid = meta.UID;
@@ -381,6 +382,14 @@ export async function checkToeStepping(records, client, baseline, options, appSh
     if (options.yes || bulkAction === 'push_all') {
       continue; // push this record
     }
+    if (bulkAction === 'pull_all') {
+      await applyServerToLocal(serverEntry, meta, metaPath, serverTz);
+      _updateBaselineEntry(baseline, entity, uid, serverEntry);
+      baselineModified = true;
+      log.success(`  Pulled server version of "${label}" to local`);
+      skippedUIDs.add(uid);
+      continue;
+    }
     if (bulkAction === 'skip_all') {
       skippedUIDs.add(uid);
       continue;
@@ -395,9 +404,11 @@ export async function checkToeStepping(records, client, baseline, options, appSh
         message: `"${label}" has server changes. How to proceed?`,
         choices: [
           { name: 'Push anyway (overwrite server changes)', value: 'push' },
+          { name: 'Pull from server (overwrite local changes)', value: 'pull' },
           { name: 'Compare differences', value: 'compare' },
           { name: 'Skip this record', value: 'skip' },
           { name: 'Push all remaining (overwrite all)', value: 'push_all' },
+          { name: 'Pull all remaining (overwrite all local)', value: 'pull_all' },
           { name: 'Skip all remaining', value: 'skip_all' },
           { name: 'Cancel entire push', value: 'cancel' },
         ],
@@ -414,6 +425,14 @@ export async function checkToeStepping(records, client, baseline, options, appSh
       log.info('Push cancelled. Run "dbo pull" to fetch server changes first.');
       return false;
     }
+    if (action === 'pull' || action === 'pull_all') {
+      await applyServerToLocal(serverEntry, meta, metaPath, serverTz);
+      _updateBaselineEntry(baseline, entity, uid, serverEntry);
+      baselineModified = true;
+      log.success(`  Pulled server version of "${label}" to local`);
+      skippedUIDs.add(uid);
+      if (action === 'pull_all') bulkAction = 'pull_all';
+    }
     if (action === 'skip' || action === 'skip_all') {
       skippedUIDs.add(uid);
       if (action === 'skip_all') bulkAction = 'skip_all';
@@ -423,6 +442,12 @@ export async function checkToeStepping(records, client, baseline, options, appSh
     }
   }
 
+  if (baselineModified) {
+    try {
+      await saveBaseline(baseline);
+    } catch { /* non-critical — next push will re-detect */ }
+  }
+
   if (!hasConflicts) return true;
 
   // Return skipped UIDs so the caller can filter them out
@@ -496,3 +521,78 @@ async function showPushDiff(serverEntry, localMeta, metaPath) {
 
   log.plain('');
 }
+
+/**
+ * Update the in-memory baseline entry for a record with the server's current
+ * values. Called after pulling from server so the next toe-stepping check
+ * sees the pulled state as the new baseline and does not re-raise the conflict.
+ *
+ * @param {Object} baseline    - Loaded baseline object (mutated in place)
+ * @param {string} entity      - Entity type (e.g., "content")
+ * @param {string} uid         - Record UID
+ * @param {Object} serverEntry - Live server record
+ */
+function _updateBaselineEntry(baseline, entity, uid, serverEntry) {
+  if (!baseline?.children) return;
+  const arr = baseline.children[entity];
+  if (!Array.isArray(arr)) return;
+  const idx = arr.findIndex(e => e.UID === uid);
+  if (idx < 0) return;
+
+  const SKIP = new Set(['_entity', '_companionReferenceColumns', '_contentColumns',
+    '_mediaFile', '_pathConfirmed', 'children', '_id']);
+  for (const [col, rawVal] of Object.entries(serverEntry)) {
+    if (SKIP.has(col)) continue;
+    const decoded = resolveContentValue(rawVal);
+    arr[idx][col] = decoded !== null ? decoded : rawVal;
+  }
+}
+
+/**
+ * Overwrite local metadata and companion content files with server values.
+ *
+ * Called when the user chooses "Pull from server" during conflict resolution.
+ * Updates:
+ *   - companion content files (columns listed in _companionReferenceColumns)
+ *   - all non-system metadata fields
+ *   - _LastUpdated / _CreatedOn timestamps in the metadata JSON
+ *   - file timestamps on both the metadata file and any companion files
+ *
+ * @param {Object} serverEntry - Live server record (from fetchServerRecord*)
+ * @param {Object} localMeta   - Currently loaded metadata object
+ * @param {string} metaPath    - Absolute path to the .metadata.json file
+ * @param {string} [serverTz]  - Server timezone string (e.g. "America/Chicago")
+ */
+async function applyServerToLocal(serverEntry, localMeta, metaPath, serverTz) {
+  const metaDir = dirname(metaPath);
+  const companions = new Set(localMeta._companionReferenceColumns || []);
+
+  // Write companion content files from server values
+  for (const col of companions) {
+    const ref = localMeta[col];
+    if (!ref || !String(ref).startsWith('@')) continue;
+    const filePath = join(metaDir, String(ref).substring(1));
+    const serverValue = resolveContentValue(serverEntry[col]);
+    if (serverValue !== null) {
+      await writeFile(filePath, serverValue, 'utf8');
+      try {
+        await setFileTimestamps(filePath, serverEntry._CreatedOn, serverEntry._LastUpdated, serverTz);
+      } catch { /* non-critical */ }
+    }
+  }
+
+  // Merge non-system, non-companion server columns into localMeta
+  const skipMeta = new Set(['_entity', '_companionReferenceColumns', '_contentColumns', '_mediaFile',
+    '_pathConfirmed', 'children', '_id']);
+  for (const [col, rawVal] of Object.entries(serverEntry)) {
+    if (skipMeta.has(col)) continue;
+    if (companions.has(col)) continue; // companion already handled as a file
+    const decoded = resolveContentValue(rawVal);
+    localMeta[col] = decoded !== null ? decoded : rawVal;
+  }
+
+  await writeFile(metaPath, JSON.stringify(localMeta, null, 2) + '\n');
+  try {
+    await setFileTimestamps(metaPath, serverEntry._CreatedOn, serverEntry._LastUpdated, serverTz);
+  } catch { /* non-critical */ }
+}

package/src/migrations/008-metadata-uid-in-suffix.js

@@ -1,7 +1,7 @@
 import { readdir, rename, access } from 'fs/promises';
 import { join, basename, dirname } from 'path';
 import { log } from '../lib/logger.js';
-import { detectLegacyTildeMetadata, buildMetaFilename } from '../lib/filenames.js';
+import { detectLegacyTildeMetadata } from '../lib/filenames.js';
 
 export const description = 'Rename metadata files from name~uid.metadata.json to name.metadata~uid.json';
 
@@ -32,7 +32,9 @@ export default async function run(_options) {
     if (!parsed) continue;
 
     const { naturalBase, uid } = parsed;
-    const newFilename = buildMetaFilename(naturalBase, uid);
+    // Migration 008 target is the intermediate name.metadata~uid.json format;
+    // migration 013 will later remove the uid suffix entirely.
+    const newFilename = `${naturalBase}.metadata~${uid}.json`;
     const newPath = join(dir, newFilename);
 
     // Skip if target already exists (avoid overwrite)

package/src/migrations/009-fix-media-collision-metadata-names.js

@@ -1,7 +1,13 @@
 import { readdir, readFile, rename, unlink, access } from 'fs/promises';
 import { join, basename, dirname, extname } from 'path';
 import { log } from '../lib/logger.js';
-import { parseMetaFilename, buildMetaFilename } from '../lib/filenames.js';
+import { parseMetaFilename } from '../lib/filenames.js';
+
+// Build the intermediate name.metadata~uid.json format this migration targets.
+// (Migration 013 will later strip the uid from filenames entirely.)
+function buildLegacySuffixFilename(naturalBase, uid) {
+  return `${naturalBase}.metadata~${uid}.json`;
+}
 
 export const description = 'Fix media collision suffix: rename (media) → _media and fix mismatched metadata filenames';
 
@@ -87,7 +93,7 @@ export default async function run(_options) {
       // Rename metadata file itself if it contains (media)
       if (parsed.naturalBase.includes('(media)')) {
         const newBase = parsed.naturalBase.replace('(media)', '_media');
-        const newMetaFilename = buildMetaFilename(newBase, parsed.uid);
+        const newMetaFilename = buildLegacySuffixFilename(newBase, parsed.uid);
         const newMetaPath = join(dirname(metaPath), newMetaFilename);
         try { await access(newMetaPath); } catch {
           await rename(metaPath, newMetaPath);
@@ -112,7 +118,7 @@ export default async function run(_options) {
       // Already correct
       if (currentParsed.naturalBase === refFilename) continue;
 
-      const correctFilename = buildMetaFilename(refFilename, currentParsed.uid);
+      const correctFilename = buildLegacySuffixFilename(refFilename, currentParsed.uid);
       const correctPath = join(dirname(metaPath), correctFilename);
 
       // If correct metadata already exists, this one is an orphan

package/src/migrations/013-remove-uid-from-meta-filenames.js

@@ -0,0 +1,117 @@
+import { readFile, writeFile, rename, readdir, access } from 'fs/promises';
+import { join, basename, dirname } from 'path';
+
+export const description = 'Rename metadata files from name.metadata~uid.json to name.metadata.json';
+
+/**
+ * Migration 013 — Remove UID from metadata filenames.
+ *
+ * Old format: colors.metadata~abc123.json
+ * New format: colors.metadata.json
+ *
+ * The UID is already stored inside the JSON as the "UID" field — no information is lost.
+ *
+ * Collision resolution uses entity priority:
+ *   content > output > everything else
+ *
+ * Within the same entity type, the first record (alphabetically by old filename) wins
+ * the unsuffixed slot; subsequent ones get -1, -2, etc.
+ *
+ * Does NOT rename companion files (they use natural names already).
+ */
+export default async function run(_options) {
+  const cwd = process.cwd();
+  let totalRenamed = 0;
+
+  const legacyFiles = await findLegacySuffixMetadataFiles(cwd);
+  if (legacyFiles.length === 0) return;
+
+  // Group by directory so we can resolve collisions per-dir
+  const byDir = new Map();
+  for (const filePath of legacyFiles) {
+    const dir = dirname(filePath);
+    if (!byDir.has(dir)) byDir.set(dir, []);
+    byDir.get(dir).push(filePath);
+  }
+
+  for (const [dir, files] of byDir) {
+    // Read entity type from each file to apply priority ordering
+    const withMeta = [];
+    for (const filePath of files) {
+      let entity = 'other';
+      let uid = null;
+      try {
+        const content = JSON.parse(await readFile(filePath, 'utf8'));
+        entity = content._entity || 'other';
+        uid = content.UID || null;
+      } catch { /* use defaults */ }
+
+      // Parse naturalBase from legacy filename: name.metadata~uid.json
+      const filename = basename(filePath);
+      const m = filename.match(/^(.+)\.metadata~([a-z0-9_]+)\.json$/i);
+      const naturalBase = m ? m[1] : filename.replace(/\.metadata~[^.]+\.json$/i, '');
+
+      withMeta.push({ filePath, naturalBase, entity, uid });
+    }
+
+    // Sort by entity priority: content first, output second, then others (alphabetically within tier)
+    const PRIORITY = { content: 0, output: 1 };
+    withMeta.sort((a, b) => {
+      const pa = PRIORITY[a.entity] ?? 2;
+      const pb = PRIORITY[b.entity] ?? 2;
+      if (pa !== pb) return pa - pb;
+      return a.naturalBase.localeCompare(b.naturalBase);
+    });
+
+    // Assign new filenames with collision resolution
+    const usedBases = new Map(); // naturalBase (lowercase) → count of times used
+
+    for (const { filePath, naturalBase, entity } of withMeta) {
+      const baseKey = naturalBase.toLowerCase();
+      const count = usedBases.get(baseKey) || 0;
+      usedBases.set(baseKey, count + 1);
+
+      const newFilename = count === 0
+        ? `${naturalBase}.metadata.json`
+        : `${naturalBase}-${count}.metadata.json`;
+
+      const newPath = join(dir, newFilename);
+
+      // Skip if target already exists (safe guard against re-running migration)
+      try { await access(newPath); continue; } catch { /* doesn't exist — safe to rename */ }
+
+      try {
+        await rename(filePath, newPath);
+        if (newFilename !== basename(filePath)) {
+          console.log(`  [${entity}] ${basename(filePath)} → ${newFilename}`);
+          totalRenamed++;
+        }
+      } catch (err) {
+        console.warn(`  (skip) Could not rename ${basename(filePath)}: ${err.message}`);
+      }
+    }
+  }
+
+  if (totalRenamed > 0) {
+    console.log(`  Renamed ${totalRenamed} metadata file(s) — UID now stored in JSON only`);
+  }
+}
+
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
+
+async function findLegacySuffixMetadataFiles(dir) {
+  const results = [];
+  try {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (SKIP.has(entry.name)) continue;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        results.push(...await findLegacySuffixMetadataFiles(full));
+      } else if (/\.metadata~[a-z0-9_]+\.json$/i.test(entry.name)) {
+        results.push(full);
+      }
+    }
+  } catch { /* skip unreadable dirs */ }
+  return results;
+}

package/src/migrations/014-entity-dir-to-data-source.js

@@ -0,0 +1,68 @@
+import { readdir, rename, mkdir, rmdir, access } from 'fs/promises';
+import { join } from 'path';
+import { log } from '../lib/logger.js';
+
+export const description = 'Move lib/entity/ files into lib/data_source/';
+
+/**
+ * Migration 014 — Relocate entity records from lib/entity/ into lib/data_source/.
+ *
+ * Entity (table-definition) records are now co-located with data source records
+ * under lib/data_source/. Files keep their _entity: "entity" metadata value —
+ * only their directory changes.
+ *
+ * If lib/data_source/ already contains a file with the same name, the source
+ * file is left in place and a warning is emitted; no data is overwritten.
+ */
+export default async function run() {
+  const cwd = process.cwd();
+  const srcDir = join(cwd, 'lib', 'entity');
+
+  // Nothing to do if lib/entity/ doesn't exist
+  try {
+    await access(srcDir);
+  } catch {
+    return;
+  }
+
+  const entries = await readdir(srcDir, { withFileTypes: true });
+  if (entries.length === 0) {
+    // Empty directory — just remove it
+    try { await rmdir(srcDir); } catch { /* ignore */ }
+    return;
+  }
+
+  const destDir = join(cwd, 'lib', 'data_source');
+  await mkdir(destDir, { recursive: true });
+
+  let movedCount = 0;
+  let skippedCount = 0;
+
+  for (const entry of entries) {
+    const srcPath = join(srcDir, entry.name);
+    const destPath = join(destDir, entry.name);
+
+    // Check for collision
+    try {
+      await access(destPath);
+      log.warn(`  Migration 014: skipped ${entry.name} — already exists in lib/data_source/`);
+      skippedCount++;
+      continue;
+    } catch { /* dest absent — safe to move */ }
+
+    await rename(srcPath, destPath);
+    movedCount++;
+  }
+
+  // Remove lib/entity/ if now empty
+  try {
+    const remaining = await readdir(srcDir);
+    if (remaining.length === 0) {
+      await rmdir(srcDir);
+    }
+  } catch { /* ignore */ }
+
+  if (movedCount > 0) {
+    log.success(`  Migration 014: moved ${movedCount} file(s) from lib/entity/ → lib/data_source/${skippedCount > 0 ? ` (${skippedCount} skipped — collision)` : ''}`);
+  }
+}