@dboio/cli 0.10.1 → 0.11.1

package/src/lib/delta.js

@@ -147,6 +147,17 @@ export async function detectChangedColumns(metaPath, baseline) {
     }
   }
 
+  // Check for columns present in baseline but removed from local metadata
+  // (user deleted the key or set it to null → should clear on server)
+  for (const [columnName, baselineVal] of Object.entries(baselineEntry)) {
+    if (shouldSkipColumn(columnName)) continue;
+    if (columnName in metadata) continue; // already compared above
+    // Baseline has a non-null value but local metadata doesn't have this key at all
+    if (baselineVal !== null && baselineVal !== undefined && normalizeValue(baselineVal) !== '') {
+      changedColumns.push(columnName);
+    }
+  }
+
   // Check _mediaFile for binary file changes (media entities)
   if (metadata._mediaFile && isReference(metadata._mediaFile)) {
     const mediaPath = resolveReferencePath(metadata._mediaFile, metaDir);
@@ -285,9 +296,90 @@ async function _detectEntityChanges(entity, baseline, metaDir) {
       }
     }
   }
+  // Check for columns removed from entity but present in baseline
+  for (const [col, baselineVal] of Object.entries(baselineEntry)) {
+    if (shouldSkipColumn(col) || col === 'children') continue;
+    if (col in entity) continue;
+    if (baselineVal !== null && baselineVal !== undefined && normalizeValue(baselineVal) !== '') {
+      changed.push(col);
+    }
+  }
   return changed;
 }
 
+// ─── Bin Delta Detection ─────────────────────────────────────────────────────
+
+/** Columns tracked for bin change detection */
+const BIN_TRACKED_COLUMNS = ['Name', 'Path', 'ParentBinID', 'Active', 'Public'];
+
+/**
+ * Detect changes between a current bin entry (from structure.json) and the
+ * baseline (from app.json children.bin array).
+ *
+ * @param {Object} binEntry - Current bin entry from structure.json (with binId, name, path, etc.)
+ * @param {Object} baseline - The baseline JSON (app.json)
+ * @returns {string[]} - Array of changed column names
+ */
+export function detectBinChanges(binEntry, baseline) {
+  if (!baseline || !baseline.children || !Array.isArray(baseline.children.bin)) {
+    return BIN_TRACKED_COLUMNS; // no baseline → treat all as changed
+  }
+
+  const baselineBin = baseline.children.bin.find(
+    b => b.UID === binEntry.uid || b.BinID === binEntry.binId
+  );
+
+  if (!baselineBin) {
+    return BIN_TRACKED_COLUMNS; // new bin, not in baseline
+  }
+
+  const changed = [];
+
+  // Map structure.json field names → server column names for comparison
+  const fieldMap = {
+    Name:        { current: binEntry.name,        baseline: baselineBin.Name },
+    Path:        { current: binEntry.path,        baseline: baselineBin.Path },
+    ParentBinID: { current: binEntry.parentBinID, baseline: baselineBin.ParentBinID },
+  };
+
+  for (const [col, { current, baseline: base }] of Object.entries(fieldMap)) {
+    const curStr = current !== null && current !== undefined ? String(current) : '';
+    const baseStr = base !== null && base !== undefined ? String(base) : '';
+    if (curStr !== baseStr) changed.push(col);
+  }
+
+  // Direct column comparisons for Active/Public (may exist on baseline)
+  for (const col of ['Active', 'Public']) {
+    if (baselineBin[col] !== undefined && binEntry[col] !== undefined) {
+      if (String(binEntry[col]) !== String(baselineBin[col])) {
+        changed.push(col);
+      }
+    }
+  }
+
+  return changed;
+}
+
+/**
+ * Synthesize a virtual metadata object for a bin entry (for use with pushFromMetadata).
+ *
+ * @param {Object} binEntry - Bin entry from structure.json
+ * @param {number} [appId] - AppID from app config
+ * @returns {Object} - Virtual metadata object
+ */
+export function synthesizeBinMetadata(binEntry, appId) {
+  const meta = {
+    _entity: 'bin',
+    _id: binEntry.binId,
+    UID: binEntry.uid,
+    Name: binEntry.name,
+    Path: binEntry.path,
+    ParentBinID: binEntry.parentBinID,
+  };
+  if (appId) meta.AppID = appId;
+  return meta;
+}
+
 async function _walkChildrenForChanges(childrenObj, baseline, metaDir, result) {
   for (const docKey of _OUTPUT_DOC_KEYS) {
     const entityArray = childrenObj[docKey];

package/src/lib/diff.js

@@ -3,9 +3,120 @@ import { readFile, writeFile, readdir, access, stat } from 'fs/promises';
 import { join, dirname, basename, extname, relative } from 'path';
 import { loadIgnore } from './ignore.js';
 import { parseServerDate, setFileTimestamps } from './timestamps.js';
-import { loadConfig, loadUserInfo } from './config.js';
+import { loadConfig, loadUserInfo, loadAppJsonBaseline } from './config.js';
+import { findBaselineEntry } from './delta.js';
 import { log } from './logger.js';
 
+// ─── Baseline Cache ─────────────────────────────────────────────────────────
+// Cached baseline used by isServerNewer to compare raw _LastUpdated strings
+// and avoid false-positive conflicts from timezone conversion drift.
+let _cachedBaseline;
+
+/**
+ * Pre-load the previous baseline into memory so that isServerNewer can
+ * do a fast string comparison.  Call once at the start of clone/pull.
+ */
+export async function loadBaselineForComparison() {
+  _cachedBaseline = await loadAppJsonBaseline();
+  return _cachedBaseline;
+}
+
+/**
+ * Clear the cached baseline.
+ * Call after saving a new baseline (e.g. after clone completes).
+ */
+export function resetBaselineCache() {
+  _cachedBaseline = undefined;
+}
+
+/**
+ * Look up the baseline _LastUpdated for a given entity/uid pair.
+ * Requires loadBaselineForComparison() to have been called first.
+ */
+function getBaselineLastUpdated(entity, uid) {
+  if (!_cachedBaseline) return undefined;
+  const entry = findBaselineEntry(_cachedBaseline, entity, uid);
+  return entry?._LastUpdated;
+}
+
+// ─── File Finding ───────────────────────────────────────────────────────────
+
+/**
+ * Recursively search the project directory for a file matching a bare filename.
+ * Respects .dboignore and skips known non-project dirs.
+ *
+ * @param {string} filename - Bare filename (e.g., "myfile.md", "myfile.metadata.json")
+ * @param {string} [dir=process.cwd()] - Starting directory
+ * @param {import('ignore').Ignore} [ig] - Ignore instance
+ * @returns {Promise<string[]>} - Array of matching absolute paths
+ */
+export async function findFileInProject(filename, dir, ig) {
+  if (!dir) dir = process.cwd();
+  if (!ig) ig = await loadIgnore();
+
+  const results = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
+      if (ig.ignores(relPath + '/') || ig.ignores(relPath)) continue;
+      results.push(...await findFileInProject(filename, fullPath, ig));
+    } else if (entry.name === filename) {
+      const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
+      if (!ig.ignores(relPath)) results.push(fullPath);
+    }
+  }
+
+  return results;
+}
+
+// ─── UID Lookup ─────────────────────────────────────────────────────────────
+
+/**
+ * Find records by UID across all metadata files and structure.json bins.
+ *
+ * @param {string[]} uids - Array of UIDs to search for
+ * @returns {Promise<Array<{ uid: string, metaPath?: string, meta?: Object, binEntry?: Object }>>}
+ */
+export async function findByUID(uids) {
+  const uidSet = new Set(uids);
+  const results = [];
+  const found = new Set();
+
+  // Search metadata files
+  const ig = await loadIgnore();
+  const metaFiles = await findMetadataFiles(process.cwd(), ig);
+
+  for (const metaPath of metaFiles) {
+    try {
+      const raw = await readFile(metaPath, 'utf8');
+      const meta = JSON.parse(raw);
+      if (meta.UID && uidSet.has(meta.UID)) {
+        results.push({ uid: meta.UID, metaPath, meta });
+        found.add(meta.UID);
+      }
+    } catch { /* skip unreadable files */ }
+  }
+
+  // Search structure.json bin entries
+  if (found.size < uidSet.size) {
+    try {
+      const { loadStructureFile } = await import('./structure.js');
+      const structure = await loadStructureFile();
+      for (const [binId, entry] of Object.entries(structure)) {
+        if (entry.uid && uidSet.has(entry.uid) && !found.has(entry.uid)) {
+          results.push({ uid: entry.uid, binEntry: { binId: Number(binId), ...entry } });
+          found.add(entry.uid);
+        }
+      }
+    } catch { /* no structure file */ }
+  }
+
+  return results;
+}
+
 // ─── Content Value Resolution ───────────────────────────────────────────────
 
 /**
@@ -30,7 +141,7 @@ async function fileExists(path) {
 
 /**
  * Recursively find all metadata files in a directory.
- * Includes .metadata.json files and output hierarchy files (_output~*.json).
+ * Includes .metadata.json files and output hierarchy files (Name~UID.json).
  */
 export async function findMetadataFiles(dir, ig) {
   if (!ig) ig = await loadIgnore();
@@ -47,11 +158,10 @@ export async function findMetadataFiles(dir, ig) {
     } else if (entry.name.endsWith('.metadata.json')) {
       const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
       if (!ig.ignores(relPath)) results.push(fullPath);
-    } else if (entry.name.startsWith('_output~') && entry.name.endsWith('.json') && !entry.name.includes('.CustomSQL.')) {
-      // Output hierarchy root files only: _output~<name>~<uid>.json
+    } else if (entry.name.endsWith('.json') && !entry.name.endsWith('.metadata.json') && !entry.name.includes('.CustomSQL.') && entry.name.includes('~')) {
+      // Output hierarchy root files: Name~UID.json (or legacy _output~Name~UID.json)
       // Exclude old-format child output files — they contain a dot-prefixed
       // child-type segment (.column~, .join~, .filter~) before .json.
-      // Root output files never have these segments, even when <name> contains dots.
       const isChildFile = /\.(column|join|filter)~/.test(entry.name);
       if (!isChildFile) {
         const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
@@ -140,11 +250,30 @@ export async function hasLocalModifications(metaPath, config = {}) {
 /**
  * Compare local sync time against server's _LastUpdated.
  * Returns true if the server record is newer than local files.
+ *
+ * When entity + uid are provided AND a baseline has been pre-loaded via
+ * loadBaselineForComparison(), the raw _LastUpdated strings are compared
+ * first.  If they match exactly, the record hasn't changed on the server
+ * and we skip the timezone conversion entirely — preventing false positives
+ * caused by DST-related rounding in the mtime ↔ parseServerDate round-trip.
+ *
+ * @param {Date}   localSyncTime      - File mtime (from getLocalSyncTime)
+ * @param {string} serverLastUpdated  - Raw _LastUpdated from the fresh server fetch
+ * @param {Object} config             - Must include ServerTimezone
+ * @param {string} [entity]           - Entity name for baseline lookup
+ * @param {string} [uid]              - Record UID for baseline lookup
  */
-export function isServerNewer(localSyncTime, serverLastUpdated, config) {
+export function isServerNewer(localSyncTime, serverLastUpdated, config, entity, uid) {
   if (!serverLastUpdated) return false;
   if (!localSyncTime) return true;
 
+  // Fast path: compare raw server string against baseline.
+  // If identical, the record hasn't changed — no timezone math needed.
+  if (entity && uid) {
+    const baselineLastUpdated = getBaselineLastUpdated(entity, uid);
+    if (baselineLastUpdated && serverLastUpdated === baselineLastUpdated) return false;
+  }
+
   const serverTz = config.ServerTimezone;
   const serverDate = parseServerDate(serverLastUpdated, serverTz);
   if (!serverDate) return false;
@@ -155,17 +284,8 @@ export function isServerNewer(localSyncTime, serverLastUpdated, config) {
 
 // ─── Server Fetching ────────────────────────────────────────────────────────
 
-/**
- * Fetch a single record from the server by entity and UID.
- */
-export async function fetchServerRecord(client, entity, uid) {
-  const result = await client.get(`/api/output/entity/${entity}/${uid}`, {
-    '_format': 'json_raw',
-  });
-  const data = result.payload || result.data;
-  const rows = Array.isArray(data) ? data : (data?.Rows || data?.rows || [data]);
-  return rows.length > 0 ? rows[0] : null;
-}
+// Per-record fetch removed — all server record fetching now goes through
+// fetchServerRecordsBatch() in toe-stepping.js via /api/app/object/.
 
 // ─── Diff Algorithm ─────────────────────────────────────────────────────────
 
@@ -327,8 +447,12 @@ function buildHunk(entries, start, end) {
 /**
  * Compare a local record (metadata.json + content files) against the server.
  * Returns a DiffResult object.
+ *
+ * @param {string} metaPath - Absolute path to the .metadata.json file
+ * @param {Object} config
+ * @param {Map<string, Object>} serverRecordsMap - Pre-fetched server records (uid → record)
  */
-export async function compareRecord(metaPath, client, config) {
+export async function compareRecord(metaPath, config, serverRecordsMap) {
   let localMeta;
   try {
     localMeta = JSON.parse(await readFile(metaPath, 'utf8'));
@@ -343,7 +467,7 @@ export async function compareRecord(metaPath, client, config) {
     return { error: `Missing _entity or UID in ${metaPath}` };
   }
 
-  const serverRecord = await fetchServerRecord(client, entity, uid);
+  const serverRecord = serverRecordsMap?.get(uid) || null;
   if (!serverRecord) {
     return { error: `Record not found on server: ${entity}/${uid}` };
   }

package/src/lib/ignore.js

@@ -16,7 +16,6 @@ const DEFAULT_FILE_CONTENT = `# DBO CLI ignore patterns
 .dboignore
 *.dboio.json
 app.json
-dbo.deploy.json
 
 # Editor / IDE / OS
 .DS_Store
@@ -43,7 +42,7 @@ package-lock.json
 
 # Local development (not pushed to DBO server)
 src/
-tests/
+test/
 
 # Documentation (repo scaffolding)
 SETUP.md

package/src/lib/metadata-templates.js

@@ -28,20 +28,37 @@ export function resolveDirective(filePath) {
   const parts = rel.split('/');
   const topDir = parts[0];
 
-  // docs/ prefix → static mapping
+  // docs/ prefix at project root → static mapping (docs/ stays at root)
   if (topDir === 'docs') {
     return { ...STATIC_DIRECTIVE_MAP.docs };
   }
 
-  // extension/<descriptor>/ — need at least 3 parts (extension/descriptor/file)
+  // lib/<subDir>/... — all server-managed dirs are now under lib/
+  if (topDir === 'lib') {
+    const subDir = parts[1];
+    if (!subDir) return null;
+
+    // lib/extension/<descriptor>/<file> — need at least 4 parts
+    if (subDir === 'extension') {
+      if (parts.length < 4) return null;
+      const descriptorDir = parts[2];
+      if (descriptorDir.startsWith('.') || descriptorDir === '_unsupported') return null;
+      return { entity: 'extension', descriptor: descriptorDir };
+    }
+
+    // lib/<entityType>/<file> — other entity dirs
+    if (ENTITY_DIR_NAMES.has(subDir)) {
+      return { entity: subDir, descriptor: null };
+    }
+  }
+
+  // Legacy fallback: bare entity-dir paths (pre-migration projects)
   if (topDir === 'extension') {
     if (parts.length < 3) return null;
     const secondDir = parts[1];
     if (secondDir.startsWith('.')) return null;
     return { entity: 'extension', descriptor: secondDir };
   }
-
-  // Other entity-dir types
   if (ENTITY_DIR_NAMES.has(topDir)) {
     return { entity: topDir, descriptor: null };
   }

package/src/lib/migrations.js

@@ -0,0 +1,75 @@
+import { readdir } from 'fs/promises';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { isInitialized, loadCompletedMigrations, saveCompletedMigration } from './config.js';
+import { log } from './logger.js';
+
+const MIGRATIONS_DIR = join(dirname(fileURLToPath(import.meta.url)), '../migrations');
+const MIGRATION_FILE_RE = /^(\d{3})-[\w-]+\.js$/;
+
+/**
+ * Run all pending migrations in sequence.
+ * Called at the start of every command handler unless options.migrate === false.
+ *
+ * @param {object} options - Command options object (checked for migrate flag)
+ */
+export async function runPendingMigrations(options = {}) {
+  // --no-migrate suppresses for this invocation
+  if (options.migrate === false) return;
+
+  // No .dbo/ project in this directory — nothing to migrate
+  if (!(await isInitialized())) return;
+
+  // Discover migration files
+  let files;
+  try {
+    files = await readdir(MIGRATIONS_DIR);
+  } catch {
+    return; // migrations/ directory absent — safe no-op
+  }
+
+  const migrationFiles = files
+    .filter(f => MIGRATION_FILE_RE.test(f))
+    .sort(); // ascending by NNN prefix
+
+  if (migrationFiles.length === 0) return;
+
+  const completed = new Set(await loadCompletedMigrations());
+  const pending = migrationFiles.filter(f => {
+    const id = f.match(MIGRATION_FILE_RE)?.[1];
+    return id && !completed.has(id);
+  });
+
+  if (pending.length === 0) return;
+
+  for (const file of pending) {
+    const id = file.match(MIGRATION_FILE_RE)[1];
+    try {
+      const migrationPath = join(MIGRATIONS_DIR, file);
+      const { default: run, description } = await import(migrationPath);
+      await run(options);
+      await saveCompletedMigration(id);
+      log.success(`Migration ${id}: ${description}`);
+    } catch (err) {
+      log.warn(`Migration ${id} failed: ${err.message} — skipped`);
+    }
+  }
+}
+
+/**
+ * Count pending (unrun) migrations without executing them.
+ * Used by `dbo status` to report pending count.
+ *
+ * @returns {Promise<number>}
+ */
+export async function countPendingMigrations() {
+  if (!(await isInitialized())) return 0;
+  let files;
+  try { files = await readdir(MIGRATIONS_DIR); } catch { return 0; }
+  const migrationFiles = files.filter(f => MIGRATION_FILE_RE.test(f));
+  const completed = new Set(await loadCompletedMigrations());
+  return migrationFiles.filter(f => {
+    const id = f.match(MIGRATION_FILE_RE)?.[1];
+    return id && !completed.has(id);
+  }).length;
+}

package/src/lib/save-to-disk.js

@@ -230,7 +230,7 @@ export async function saveToDisk(rows, columns, options = {}) {
 
       if (bulkAction !== 'overwrite_all') {
         const localSyncTime = await diffModule.getLocalSyncTime(metaPath);
-        const serverIsNewer = diffModule.isServerNewer(localSyncTime, row._LastUpdated, options.config || {});
+        const serverIsNewer = diffModule.isServerNewer(localSyncTime, row._LastUpdated, options.config || {}, options.entity, row.UID);
 
         if (serverIsNewer) {
           const action = await diffModule.promptChangeDetection(finalName, row, options.config || {});

package/src/lib/scaffold.js

@@ -1,6 +1,6 @@
 import { mkdir, stat, readFile, writeFile, access } from 'fs/promises';
 import { join } from 'path';
-import { DEFAULT_PROJECT_DIRS } from './structure.js';
+import { SCAFFOLD_DIRS } from './structure.js';
 import { log } from './logger.js';
 import { applyTrashIcon } from './folder-icon.js';
 
@@ -9,20 +9,15 @@ import { applyTrashIcon } from './folder-icon.js';
  * Creates missing directories, skips existing ones, warns on name conflicts.
  * Also creates app.json with {} if absent.
  *
- * When appShortName is provided, also creates media sub-directories:
- *   media/<appShortName>/app/
- *   media/<appShortName>/user/
- *
  * @param {string} [cwd=process.cwd()]
- * @param {{ appShortName?: string }} [options]
  * @returns {Promise<{ created: string[], skipped: string[], warned: string[] }>}
  */
-export async function scaffoldProjectDirs(cwd = process.cwd(), options = {}) {
+export async function scaffoldProjectDirs(cwd = process.cwd()) {
   const created = [];
   const skipped = [];
   const warned  = [];
 
-  for (const dir of DEFAULT_PROJECT_DIRS) {
+  for (const dir of SCAFFOLD_DIRS) {
     const target = join(cwd, dir);
     try {
       const s = await stat(target);
@@ -39,26 +34,6 @@ export async function scaffoldProjectDirs(cwd = process.cwd(), options = {}) {
     }
   }
 
-  // Create media sub-directories when app short name is known:
-  //   media/<appShortName>/app/   — app-level media assets
-  //   media/<appShortName>/user/  — user-uploaded media
-  if (options.appShortName) {
-    const mediaSubs = [
-      `media/${options.appShortName}/app`,
-      `media/${options.appShortName}/user`,
-    ];
-    for (const sub of mediaSubs) {
-      const target = join(cwd, sub);
-      try {
-        await stat(target);
-        skipped.push(sub);
-      } catch {
-        await mkdir(target, { recursive: true });
-        created.push(sub);
-      }
-    }
-  }
-
   // Best-effort: apply trash icon to the trash directory
   await applyTrashIcon(join(cwd, 'trash'));