@dboio/cli 0.15.2 → 0.16.2

package/src/lib/toe-stepping.js

@@ -5,6 +5,7 @@ import { findBaselineEntry, shouldSkipColumn, normalizeValue, isReference, resol
 import { resolveContentValue } from '../commands/clone.js';
 import { computeLineDiff, formatDiff } from './diff.js';
 import { parseMetaFilename } from './filenames.js';
+import { parseServerDate } from './timestamps.js';
 import { log } from './logger.js';
 
 /**
@@ -285,10 +286,11 @@ function findOldestBaselineDate(records, baseline) {
  * @param {Object} baseline - Loaded baseline from .dbo/.app_baseline.json
  * @param {Object} options  - Commander options (options.yes used for auto-accept)
  * @param {string} [appShortName] - App short name for bulk fetch (optional)
+ * @param {string} [serverTz] - Server timezone from config (e.g. "America/Chicago")
  * @returns {Promise<boolean|Set<string>>} - true = proceed with all,
  *   false = user cancelled entirely, Set<string> = UIDs to skip (proceed with rest)
  */
-export async function checkToeStepping(records, client, baseline, options, appShortName) {
+export async function checkToeStepping(records, client, baseline, options, appShortName, serverTz) {
   // Build list of records to check (skip new records without UID)
   const requests = [];
   for (const { meta } of records) {
@@ -350,9 +352,10 @@ export async function checkToeStepping(records, client, baseline, options, appSh
 
     if (!serverTs || !baselineTs) continue; // missing timestamps — skip safely
 
-    // Parse both as dates (ISO 8601 strings or server-format timestamps)
-    const serverDate = new Date(serverTs);
-    const baselineDate = new Date(baselineTs);
+    // Parse both dates using server timezone from config — server dates
+    // may arrive without Z suffix and represent the server's local time
+    const serverDate = parseServerDate(serverTs, serverTz);
+    const baselineDate = parseServerDate(baselineTs, serverTz);
 
     if (isNaN(serverDate) || isNaN(baselineDate)) continue; // unparseable — skip
     if (serverDate <= baselineDate) continue; // server is same or older — no conflict
@@ -440,7 +443,7 @@ export async function checkToeStepping(records, client, baseline, options, appSh
  */
 async function showPushDiff(serverEntry, localMeta, metaPath) {
   const metaDir = dirname(metaPath);
-  const contentCols = localMeta._contentColumns || [];
+  const contentCols = localMeta._companionReferenceColumns || localMeta._contentColumns || [];
 
   // Compare content file columns
   for (const col of contentCols) {
@@ -472,7 +475,7 @@ async function showPushDiff(serverEntry, localMeta, metaPath) {
   }
 
   // Compare metadata fields
-  const skipFields = new Set(['_entity', '_contentColumns', '_mediaFile', 'children', '_pathConfirmed']);
+  const skipFields = new Set(['_entity', '_contentColumns', '_companionReferenceColumns', '_mediaFile', 'children', '_pathConfirmed']);
   for (const col of Object.keys(serverEntry)) {
     if (skipFields.has(col)) continue;
     if (contentCols.includes(col)) continue;

package/src/migrations/007-natural-entity-companion-filenames.js

@@ -1,4 +1,4 @@
-import { readdir, readFile, writeFile, rename, access, mkdir } from 'fs/promises';
+import { readdir, readFile, writeFile, rename, access, mkdir, stat, utimes } from 'fs/promises';
 import { join, basename, dirname } from 'path';
 import { log } from '../lib/logger.js';
 import { stripUidFromFilename, hasUidInFilename, isMetadataFile } from '../lib/filenames.js';
@@ -102,10 +102,13 @@ export default async function run(_options) {
         claimedNaturals.add(naturalPath);
       }
 
-      // Rewrite metadata if @references were updated
+      // Rewrite metadata if @references were updated (preserve timestamps
+      // so the write doesn't cause false "local changes" during clone/pull)
       if (metaChanged) {
         try {
+          const before = await stat(metaPath);
           await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+          await utimes(metaPath, before.atime, before.mtime);
           totalRefsUpdated++;
         } catch { /* non-critical */ }
       }

package/src/migrations/009-fix-media-collision-metadata-names.js

@@ -0,0 +1,161 @@
+import { readdir, readFile, rename, unlink, access } from 'fs/promises';
+import { join, basename, dirname, extname } from 'path';
+import { log } from '../lib/logger.js';
+import { parseMetaFilename, buildMetaFilename } from '../lib/filenames.js';
+
+export const description = 'Fix media collision suffix: rename (media) → _media and fix mismatched metadata filenames';
+
+/**
+ * Migration 009 — Fix media metadata/companion files with wrong collision suffix.
+ *
+ * Two issues addressed:
+ *
+ * 1. Metadata filename mismatch: media records with collision-resolved names had their
+ *    metadata created as "env.js.metadata~uid.json" instead of matching the companion.
+ *    Detection: _mediaFile @reference base differs from metadata filename base.
+ *
+ * 2. (media) → _media rename: the old "(media)" suffix causes zsh glob issues.
+ *    Renames both companion files and metadata files, and updates @references.
+ */
+export default async function run(_options) {
+  const cwd = process.cwd();
+  let totalFixed = 0;
+  let totalOrphansDeleted = 0;
+  let totalParenRenamed = 0;
+
+  const metaFiles = await findAllMetadataFiles(cwd);
+  if (metaFiles.length === 0) return;
+
+  for (let metaPath of metaFiles) {
+    try {
+      let content = JSON.parse(await readFile(metaPath, 'utf8'));
+      const filename = basename(metaPath);
+      const parsed = parseMetaFilename(filename);
+      if (!parsed) continue;
+
+      // ── Phase 1: Rename (media) → _media in companion files and @references ──
+
+      // Check media @reference for (media) pattern
+      if (content._entity === 'media' && content._mediaFile) {
+        const ref = String(content._mediaFile);
+        if (ref.startsWith('@') && ref.includes('(media)')) {
+          const oldCompanion = ref.substring(1);
+          const ext = extname(oldCompanion);
+          const base = basename(oldCompanion, ext);
+          const newCompanion = `${base.replace('(media)', '_media')}${ext}`;
+          const dir = dirname(metaPath);
+
+          // Rename companion file on disk (or delete old if new already exists)
+          const oldCompanionPath = join(dir, oldCompanion);
+          const newCompanionPath = join(dir, newCompanion);
+          try {
+            await access(oldCompanionPath);
+            let newExists = false;
+            try { await access(newCompanionPath); newExists = true; } catch { /* */ }
+            if (newExists) {
+              // New file already exists (from re-clone) — delete the old orphan
+              await unlink(oldCompanionPath);
+              log.dim(`  Deleted old companion: ${oldCompanion}`);
+              totalOrphansDeleted++;
+            } else {
+              await rename(oldCompanionPath, newCompanionPath);
+              log.dim(`  ${oldCompanion} → ${newCompanion}`);
+            }
+          } catch { /* companion doesn't exist or already renamed */ }
+
+          // Update @reference in metadata
+          content._mediaFile = `@${newCompanion}`;
+          await import('fs/promises').then(fs => fs.writeFile(metaPath, JSON.stringify(content, null, 2) + '\n'));
+          totalParenRenamed++;
+        }
+      }
+
+      // Check content @references for (media) — shouldn't happen but be safe
+      if (content._contentColumns) {
+        for (const col of content._contentColumns) {
+          const ref = content[col];
+          if (ref && String(ref).startsWith('@') && String(ref).includes('(media)')) {
+            const oldName = String(ref).substring(1);
+            const ext = extname(oldName);
+            const base = basename(oldName, ext);
+            content[col] = `@${base.replace('(media)', '_media')}${ext}`;
+            await import('fs/promises').then(fs => fs.writeFile(metaPath, JSON.stringify(content, null, 2) + '\n'));
+          }
+        }
+      }
+
+      // Rename metadata file itself if it contains (media)
+      if (parsed.naturalBase.includes('(media)')) {
+        const newBase = parsed.naturalBase.replace('(media)', '_media');
+        const newMetaFilename = buildMetaFilename(newBase, parsed.uid);
+        const newMetaPath = join(dirname(metaPath), newMetaFilename);
+        try { await access(newMetaPath); } catch {
+          await rename(metaPath, newMetaPath);
+          log.dim(`  ${filename} → ${newMetaFilename}`);
+          metaPath = newMetaPath; // update for phase 2
+          totalParenRenamed++;
+        }
+        // Re-read parsed after rename
+        content = JSON.parse(await readFile(metaPath, 'utf8'));
+      }
+
+      // ── Phase 2: Fix metadata filename not matching @reference ──
+
+      if (content._entity !== 'media') continue;
+      const mediaRef = content._mediaFile;
+      if (!mediaRef || !String(mediaRef).startsWith('@')) continue;
+
+      const refFilename = String(mediaRef).substring(1);
+      const currentParsed = parseMetaFilename(basename(metaPath));
+      if (!currentParsed) continue;
+
+      // Already correct
+      if (currentParsed.naturalBase === refFilename) continue;
+
+      const correctFilename = buildMetaFilename(refFilename, currentParsed.uid);
+      const correctPath = join(dirname(metaPath), correctFilename);
+
+      // If correct metadata already exists, this one is an orphan
+      try {
+        await access(correctPath);
+        await unlink(metaPath);
+        log.dim(`  Deleted orphan: ${basename(metaPath)}`);
+        totalOrphansDeleted++;
+        continue;
+      } catch { /* correct file doesn't exist — rename */ }
+
+      await rename(metaPath, correctPath);
+      log.dim(`  ${basename(metaPath)} → ${correctFilename}`);
+      totalFixed++;
+    } catch { /* skip unreadable files */ }
+  }
+
+  if (totalParenRenamed > 0) {
+    log.dim(`  Renamed ${totalParenRenamed} file(s) from (media) to _media`);
+  }
+  if (totalFixed > 0) {
+    log.dim(`  Renamed ${totalFixed} media metadata file(s) to match companion names`);
+  }
+  if (totalOrphansDeleted > 0) {
+    log.dim(`  Deleted ${totalOrphansDeleted} orphaned media metadata file(s)`);
+  }
+}
+
+const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+
+async function findAllMetadataFiles(dir) {
+  const results = [];
+  try {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (SKIP.has(entry.name)) continue;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        results.push(...await findAllMetadataFiles(full));
+      } else if (entry.name.includes('.metadata~') && entry.name.endsWith('.json')) {
+        results.push(full);
+      }
+    }
+  } catch { /* skip unreadable dirs */ }
+  return results;
+}

package/src/migrations/010-delete-paren-media-orphans.js

@@ -0,0 +1,61 @@
+import { readdir, unlink, access } from 'fs/promises';
+import { join, basename, extname } from 'path';
+import { log } from '../lib/logger.js';
+
+export const description = 'Delete orphaned (media) companion and metadata files replaced by _media';
+
+/**
+ * Migration 010 — Clean up leftover (media) files.
+ *
+ * Migration 009 renamed (media) → _media but did not delete the old files
+ * when the new _media versions already existed. This migration finds any
+ * remaining files with "(media)" in their name and deletes them if the
+ * corresponding _media version exists.
+ */
+export default async function run(_options) {
+  const cwd = process.cwd();
+  let totalDeleted = 0;
+
+  const parenFiles = await findParenMediaFiles(cwd);
+  if (parenFiles.length === 0) return;
+
+  for (const filePath of parenFiles) {
+    const filename = basename(filePath);
+    // Build the _media equivalent
+    const newFilename = filename.replace(/\(media\)/g, '_media');
+    const newPath = join(filePath, '..', newFilename);
+
+    // Only delete if the _media replacement exists
+    try {
+      await access(newPath);
+      await unlink(filePath);
+      log.dim(`  Deleted: ${filename}`);
+      totalDeleted++;
+    } catch {
+      // No _media replacement — leave alone (might be a legitimate filename)
+    }
+  }
+
+  if (totalDeleted > 0) {
+    log.dim(`  Removed ${totalDeleted} orphaned (media) file(s)`);
+  }
+}
+
+const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+
+async function findParenMediaFiles(dir) {
+  const results = [];
+  try {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (SKIP.has(entry.name)) continue;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        results.push(...await findParenMediaFiles(full));
+      } else if (entry.name.includes('(media)')) {
+        results.push(full);
+      }
+    }
+  } catch { /* skip unreadable dirs */ }
+  return results;
+}

package/src/migrations/011-schema-driven-metadata.js

@@ -0,0 +1,120 @@
+import { readFile, writeFile, rename, readdir, rm, utimes } from 'fs/promises';
+import { join } from 'path';
+import { stat } from 'fs/promises';
+
+export const description = 'Rename metadata_templates.json → metadata_schema.json; migrate _contentColumns → _companionReferenceColumns; remove _unsupported dir; retire Extension_* config keys';
+
+export default async function run(_options) {
+  const log = (...args) => console.log(...args);
+
+  // ── Step 1: Rename .dbo/metadata_templates.json → .dbo/metadata_schema.json ──
+  try {
+    const oldPath = '.dbo/metadata_templates.json';
+    const newPath = '.dbo/metadata_schema.json';
+    if (await fileExists(oldPath) && !(await fileExists(newPath))) {
+      await rename(oldPath, newPath);
+      log(`  Renamed .dbo/metadata_templates.json → .dbo/metadata_schema.json`);
+    } else if (await fileExists(newPath)) {
+      log(`  .dbo/metadata_schema.json already exists — skipping rename`);
+    }
+  } catch (e) {
+    log(`  (skip) metadata_templates rename: ${e.message}`);
+  }
+
+  // ── Step 2: Rename _contentColumns → _companionReferenceColumns in all metadata files ──
+  try {
+    const metaFiles = await findMetadataFiles('.');
+    let updated = 0;
+    for (const file of metaFiles) {
+      try {
+        const raw = await readFile(file, 'utf8');
+        const obj = JSON.parse(raw);
+        if ('_contentColumns' in obj && !('_companionReferenceColumns' in obj)) {
+          // Preserve file timestamps so the write doesn't cause false
+          // "local changes" detections during the subsequent clone/pull.
+          const before = await stat(file);
+          obj._companionReferenceColumns = obj._contentColumns;
+          delete obj._contentColumns;
+          await writeFile(file, JSON.stringify(obj, null, 2) + '\n');
+          await utimes(file, before.atime, before.mtime);
+          updated++;
+        }
+      } catch { /* skip malformed files */ }
+    }
+    if (updated > 0) log(`  Migrated _contentColumns in ${updated} metadata file(s)`);
+  } catch (e) {
+    log(`  (skip) _contentColumns rename: ${e.message}`);
+  }
+
+  // ── Step 3: Move lib/extension/_unsupported/* → lib/extension/ ──
+  try {
+    const unsupportedDir = 'lib/extension/_unsupported';
+    if (await fileExists(unsupportedDir)) {
+      const files = await readdir(unsupportedDir);
+      let moved = 0;
+      for (const file of files) {
+        const src = join(unsupportedDir, file);
+        const dst = join('lib/extension', file);
+        try {
+          await rename(src, dst);
+          moved++;
+        } catch (e) {
+          log(`  (warn) Could not move ${file}: ${e.message}`);
+        }
+      }
+      try {
+        await rm(unsupportedDir, { recursive: true, force: true });
+      } catch { /* ignore */ }
+      if (moved > 0) log(`  Moved ${moved} file(s) from lib/extension/_unsupported/ to lib/extension/`);
+    }
+  } catch (e) {
+    log(`  (skip) _unsupported relocation: ${e.message}`);
+  }
+
+  // ── Step 4: Remove retired config.json keys ──
+  try {
+    const configPath = '.dbo/config.json';
+    const raw = await readFile(configPath, 'utf8');
+    const config = JSON.parse(raw);
+    const retiredPattern = /^Extension_.+_(FilenameCol|ContentExtractions)$|^ExtensionFilenameCol$/;
+    const keysToRemove = Object.keys(config).filter(k => retiredPattern.test(k));
+    if (keysToRemove.length > 0) {
+      for (const k of keysToRemove) delete config[k];
+      await writeFile(configPath, JSON.stringify(config, null, 2) + '\n');
+      log(`  Removed ${keysToRemove.length} retired config key(s): ${keysToRemove.join(', ')}`);
+    }
+  } catch (e) {
+    log(`  (skip) config key retirement: ${e.message}`);
+  }
+}
+
+async function fileExists(p) {
+  try {
+    await stat(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function findMetadataFiles(dir) {
+  const results = [];
+  async function walk(d) {
+    let entries;
+    try {
+      entries = await readdir(d, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = join(d, entry.name);
+      if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'trash') {
+        await walk(full);
+      } else if (entry.isFile() && /\.metadata~[^.]+\.json$/.test(entry.name)) {
+        results.push(full);
+      }
+    }
+  }
+  await walk(dir);
+  return results;
+}