@dboio/cli 0.9.8 → 0.11.1

package/src/lib/structure.js

@@ -3,23 +3,55 @@ import { join } from 'path';
 
 const STRUCTURE_FILE = '.dbo/structure.json';
 
-/** All bin-placed files go under this directory at project root */
-export const BINS_DIR = 'bins';
+/** All server-managed directories live under this subdirectory */
+export const LIB_DIR = 'lib';
 
-/** Default top-level directories created at project root during clone */
-export const DEFAULT_PROJECT_DIRS = [
-  'bins',
-  'automation',
-  'app_version',
+/** All bin-placed files go under this directory */
+export const BINS_DIR = 'lib/bins';
+
+/**
+ * Directories always created by `dbo init --scaffold` and `dbo clone`
+ * regardless of whether records exist for that entity type.
+ */
+export const SCAFFOLD_DIRS = [
+  'lib/bins',
+  'lib/automation',
+  'lib/app_version',
+  'lib/entity',
+  'lib/entity_column',
+  'lib/entity_column_value',
+  'lib/extension',
+  'lib/integration',
+  'lib/security',
+  'lib/security_column',
+  'src',
+  'test',
+  'trash',
   'docs',
-  'site',
-  'extension',
+];
+
+/**
+ * Entity names whose lib/<name> directories are only created on-demand
+ * when records of that entity type actually exist (during clone/pull).
+ * These are NOT created by scaffold — only by processEntityDirEntries().
+ */
+export const ON_DEMAND_ENTITY_DIRS = new Set([
   'data_source',
   'group',
-  'integration',
-  'src',
-  'tests',
-  'trash',
+  'site',
+  'redirect',
+]);
+
+/**
+ * Union of SCAFFOLD_DIRS and on-demand dirs.
+ * Used for membership checks ("is this path a known project directory?").
+ */
+export const DEFAULT_PROJECT_DIRS = [
+  ...SCAFFOLD_DIRS,
+  'lib/data_source',
+  'lib/group',
+  'lib/redirect',
+  'lib/site',
 ];
 
 /** Map from physical output table names → documentation/display names */
@@ -40,15 +72,103 @@ export const OUTPUT_HIERARCHY_ENTITIES = [
 
 /** Entity keys that correspond to project directories (key IS the dir name) */
 export const ENTITY_DIR_NAMES = new Set([
-  'extension',
+  'automation',
   'app_version',
   'data_source',
-  'site',
+  'entity',
+  'entity_column',
+  'entity_column_value',
+  'extension',
   'group',
   'integration',
-  'automation',
+  'redirect',
+  'security',
+  'security_column',
+  'site',
 ]);
 
+/**
+ * Resolve the local directory path for an entity-dir type.
+ * Always returns "lib/<entityName>" (e.g. "lib/extension", "lib/site").
+ * Use this instead of bare entity name concatenation everywhere.
+ *
+ * @param {string} entityName - Entity key from ENTITY_DIR_NAMES (e.g. "extension")
+ * @returns {string} - Path relative to project root (e.g. "lib/extension")
+ */
+export function resolveEntityDirPath(entityName) {
+  return `${LIB_DIR}/${entityName}`;
+}
+
+// ─── Core Asset Entity Classification ─────────────────────────────────────
+//
+// Core assets are entities that carry a UID, are tracked by index triggers
+// (AFTER INSERT/UPDATE/DELETE → index_insert_*), and participate in the
+// versioning / revision system.
+//
+// "Exportable" assets are part of an app's export package — a builder creates
+// and manages these.  "Non-exportable" assets have UIDs and index triggers
+// but are system-managed or instance-specific (not included in app exports).
+//
+// "Data" entities (user, audit, authentication, message, etc.) do NOT carry
+// auto-generated UIDs and are NOT tracked by index triggers.
+
+/**
+ * Exportable core asset entities (20).
+ * Part of app export packages. Builder-managed. All have UID (NOT NULL, UNIQUE)
+ * and index triggers.
+ */
+export const EXPORTABLE_ENTITIES = [
+  'app',
+  'automation',
+  'bin',
+  'content',
+  'data_source',
+  'entity',
+  'entity_column',
+  'entity_column_value',
+  'extension',
+  'group',
+  'integration',
+  'media',
+  'output',
+  'output_value',
+  'output_value_entity_column_rel',
+  'output_value_filter',
+  'redirect',
+  'security',
+  'security_column',
+  'site',
+];
+
+/**
+ * Non-exportable core asset entities (3).
+ * Have UIDs and index triggers but are NOT part of app export packages.
+ * System-managed or instance-specific.
+ */
+export const NON_EXPORTABLE_ENTITIES = [
+  'app_version',
+  'mail_server',
+  'media_server',
+];
+
+/** All core asset entities (exportable + non-exportable). 23 total. */
+export const CORE_ENTITIES = [...EXPORTABLE_ENTITIES, ...NON_EXPORTABLE_ENTITIES];
+
+/** Set variant for O(1) lookups */
+export const CORE_ENTITIES_SET = new Set(CORE_ENTITIES);
+export const EXPORTABLE_ENTITIES_SET = new Set(EXPORTABLE_ENTITIES);
+export const NON_EXPORTABLE_ENTITIES_SET = new Set(NON_EXPORTABLE_ENTITIES);
+
+/** Check whether an entity name is a core asset */
+export function isCoreEntity(entityName) {
+  return CORE_ENTITIES_SET.has(entityName);
+}
+
+/** Check whether an entity name is an exportable core asset */
+export function isExportableEntity(entityName) {
+  return EXPORTABLE_ENTITIES_SET.has(entityName);
+}
+
 /**
  * Build a bin hierarchy from an array of bin objects.
  * Filters by targetAppId and resolves full directory paths via ParentBinID traversal.
@@ -70,6 +190,21 @@ export function buildBinHierarchy(bins, targetAppId) {
   // Build lookup by BinID
   const byId = {};
   for (const bin of filtered) {
+    // bin.Name=null is legacy — these bins map directly to bins/ root.
+    // Never split bin.Path into sub-directories for legacy null-Name bins.
+    if (!bin.Name) {
+      byId[bin.BinID] = {
+        name: null,
+        path: bin.Path,
+        segment: '',
+        parentBinID: null,
+        binId: bin.BinID,
+        uid: bin.UID,
+        fullPath: '',
+      };
+      continue;
+    }
+
     // Use Path as the segment name, but only the last part if it contains slashes
     // (Path often stores the full path, e.g. "assets/css/vendor" for a bin named "vendor")
     const rawPath = bin.Path || bin.Name;
@@ -114,7 +249,9 @@ export function buildBinHierarchy(bins, targetAppId) {
  */
 export function resolveBinPath(binId, structure) {
   const entry = structure[binId];
-  return entry ? `${BINS_DIR}/${entry.fullPath}` : null;
+  if (!entry) return null;
+  // Legacy bins (Name=null) have empty fullPath → resolve to bins/ root
+  return entry.fullPath ? `${BINS_DIR}/${entry.fullPath}` : BINS_DIR;
 }
 
 /**
@@ -187,12 +324,12 @@ export function findChildBins(binId, structure) {
 // ─── Extension Descriptor Sub-directory Support ───────────────────────────
 
 /** Root for all extension descriptor-grouped sub-directories */
-export const EXTENSION_DESCRIPTORS_DIR = 'extension';
+export const EXTENSION_DESCRIPTORS_DIR = 'lib/extension';
 
-/** Extensions that cannot be mapped go here (always created, even if empty) */
-export const EXTENSION_UNSUPPORTED_DIR = 'extension/_unsupported';
+/** Extensions that cannot be mapped go here */
+export const EXTENSION_UNSUPPORTED_DIR = 'lib/extension/_unsupported';
 
-/** Root-level documentation directory for alternate placement */
+/** Root-level documentation directory (remains at project root) */
 export const DOCUMENTATION_DIR = 'docs';
 
 /**

package/src/lib/toe-stepping.js

@@ -0,0 +1,381 @@
+import { dirname, basename } from 'path';
+import { readFile } from 'fs/promises';
+import { findBaselineEntry, shouldSkipColumn, normalizeValue, isReference, resolveReferencePath } from './delta.js';
+import { resolveContentValue } from '../commands/clone.js';
+import { log } from './logger.js';
+
+/**
+ * Columns to include in the diff even though shouldSkipColumn() would normally
+ * skip them.  _LastUpdated and _LastUpdatedUserID are valuable conflict
+ * indicators: _LastUpdated reveals a newer server edit, and _LastUpdatedUserID
+ * identifies *who* made it.
+ */
+const DIFF_ALLOW = new Set(['_LastUpdated', '_LastUpdatedUserID']);
+
+/**
+ * Fetch a single record from the server by entity name + row UID.
+ *
+ * Uses the lightweight entity endpoint:
+ *   GET /api/o/e/{entity}/{rowUID}?_template=json_raw
+ *
+ * Returns the parsed record object, or null on failure (network, 404, auth).
+ *
+ * @param {DboClient} client
+ * @param {string} entity - Entity name (e.g., "content", "media", "output")
+ * @param {string} uid    - Row UID
+ * @returns {Promise<Object|null>}
+ */
+export async function fetchServerRecord(client, entity, uid) {
+  try {
+    const result = await client.get(`/api/o/e/${entity}/${uid}?_template=json_raw`);
+    if (!result.ok) return null;
+
+    // json_raw returns { Rows: [...] } or { rows: [...] } or an array
+    const data = result.payload || result.data;
+    if (Array.isArray(data)) return data.length > 0 ? data[0] : null;
+    if (data?.Rows?.length > 0) return data.Rows[0];
+    if (data?.rows?.length > 0) return data.rows[0];
+    // Direct object with UID → single record response
+    if (data && typeof data === 'object' && data.UID) return data;
+    return null;
+  } catch {
+    return null; // network or parse failure — degrade gracefully
+  }
+}
+
+/**
+ * Fetch multiple server records in parallel.
+ *
+ * Each record is fetched independently via fetchServerRecord().
+ * Uses Promise.allSettled() so one failure doesn't block others.
+ *
+ * @param {DboClient} client
+ * @param {Array<{ entity: string, uid: string }>} requests
+ * @returns {Promise<Map<string, Object>>} - Map of uid → server record (only successful fetches)
+ */
+export async function fetchServerRecords(client, requests) {
+  const results = await Promise.allSettled(
+    requests.map(({ entity, uid }) =>
+      fetchServerRecord(client, entity, uid).then(record => ({ uid, record }))
+    )
+  );
+
+  const map = new Map();
+  for (const result of results) {
+    if (result.status === 'fulfilled' && result.value.record) {
+      map.set(result.value.uid, result.value.record);
+    }
+  }
+  return map;
+}
+
+/**
+ * Fetch server records in bulk using the app object endpoint with UpdatedAfter
+ * filtering. This is far more efficient than per-record fetches because it
+ * makes a single HTTP request and the server only returns records modified
+ * after the given date.
+ *
+ * The response is used ONLY for comparison — it must NOT replace app.json or
+ * app_baseline.json.
+ *
+ * @param {DboClient} client
+ * @param {string} appShortName - App short name for the /api/app/object/ endpoint
+ * @param {string} updatedAfter - ISO date string (oldest _LastUpdated among records to push)
+ * @returns {Promise<Map<string, Object>>} - Map of uid → server record (across all entities)
+ */
+export async function fetchServerRecordsBatch(client, appShortName, updatedAfter) {
+  const map = new Map();
+
+  try {
+    // Void cache first so the app object response reflects the latest server state
+    await client.voidCache();
+
+    const result = await client.get(`/api/app/object/${appShortName}`, {
+      UpdatedAfter: updatedAfter,
+    });
+
+    if (!result.ok && !result.successful) return map;
+
+    const data = result.payload || result.data;
+    if (!data) return map;
+
+    // Normalize: the response may be a single app object or wrapped in an array
+    let appRecord;
+    if (Array.isArray(data)) {
+      appRecord = data.length > 0 ? data[0] : null;
+    } else if (data?.Rows?.length > 0) {
+      appRecord = data.Rows[0];
+    } else if (data && typeof data === 'object' && (data.UID || data.children)) {
+      appRecord = data;
+    } else {
+      return map;
+    }
+
+    if (!appRecord?.children) return map;
+
+    // Walk children hierarchy and index every record by UID
+    _indexChildren(appRecord.children, map);
+  } catch {
+    // degrade gracefully — caller will fall back to per-record fetches
+  }
+
+  return map;
+}
+
+/**
+ * Recursively index all records in a children hierarchy into a Map<UID, record>.
+ * Handles nested children (e.g. output records contain children.output_value).
+ */
+function _indexChildren(children, map) {
+  for (const [, entityArray] of Object.entries(children)) {
+    if (!Array.isArray(entityArray)) continue;
+    for (const record of entityArray) {
+      if (record.UID) {
+        map.set(record.UID, record);
+      }
+      // Recurse into nested children (e.g. output → output_value)
+      if (record.children) {
+        _indexChildren(record.children, map);
+      }
+    }
+  }
+}
+
+/**
+ * Decode a server column value that may be base64-encoded.
+ * The json_raw format may return large text as:
+ *   { bytes: N, value: "base64string", encoding: "base64" }
+ * Delegates to resolveContentValue() from clone.js.
+ *
+ * @param {*} value
+ * @returns {string|null}
+ */
+function decodeServerValue(value) {
+  return resolveContentValue(value);
+}
+
+/**
+ * Compare a server record against local metadata and baseline to
+ * build a diff description.
+ *
+ * Returns an array of conflict columns:
+ * [{ col, serverValue, localValue, baselineValue }]
+ *
+ * A "conflict column" is one where the server value differs from the baseline
+ * (meaning the server changed since we last pulled/pushed).
+ *
+ * _LastUpdated and _LastUpdatedUserID are intentionally included in the diff
+ * because they carry meaningful conflict information: _LastUpdated indicates
+ * a newer server edit, and _LastUpdatedUserID identifies who made the change.
+ *
+ * @param {Object} serverEntry    - Record from per-record server fetch
+ * @param {Object} baselineEntry  - Record from .app_baseline.json
+ * @param {Object} localMeta      - Local .metadata.json object
+ * @param {string} metaDir        - Absolute directory of the metadata file
+ * @returns {Promise<Array<{ col: string, serverValue: string, localValue: string, baselineValue: string }>>}
+ */
+export async function buildRecordDiff(serverEntry, baselineEntry, localMeta, metaDir) {
+  const conflicts = [];
+
+  // Compare all columns present in the server entry
+  for (const [col, rawServerVal] of Object.entries(serverEntry)) {
+    // Allow _LastUpdated and _LastUpdatedUserID through; skip other system cols
+    if (!DIFF_ALLOW.has(col) && shouldSkipColumn(col)) continue;
+
+    const serverValue = normalizeValue(decodeServerValue(rawServerVal));
+    const baselineValue = normalizeValue(baselineEntry ? baselineEntry[col] : undefined);
+
+    // If server value equals baseline value, no server-side change for this col
+    if (serverValue === baselineValue) continue;
+
+    // Server changed this column — record what's local too
+    const localRaw = localMeta[col];
+    let localValue;
+    if (localRaw && isReference(String(localRaw))) {
+      // Read file content for display (truncate for readability)
+      try {
+        const refPath = resolveReferencePath(String(localRaw), metaDir);
+        const content = await readFile(refPath, 'utf8');
+        localValue = content.substring(0, 120) + (content.length > 120 ? '…' : '');
+      } catch {
+        localValue = String(localRaw);
+      }
+    } else {
+      localValue = normalizeValue(localRaw);
+    }
+
+    conflicts.push({ col, serverValue: serverValue.substring(0, 120), localValue, baselineValue: baselineValue.substring(0, 120) });
+  }
+
+  return conflicts;
+}
+
+/**
+ * Print a conflict summary for a record to the terminal.
+ *
+ * @param {string} label           - Human-readable record label (filename without extension)
+ * @param {string} serverUser      - _LastUpdatedUserID from server
+ * @param {string} serverTimestamp - _LastUpdated from server
+ * @param {Array}  diffColumns     - Output of buildRecordDiff()
+ */
+export function displayConflict(label, serverUser, serverTimestamp, diffColumns) {
+  log.warn('');
+  log.warn(`  Server conflict: "${label}"`);
+  log.label('     Changed by', serverUser || 'unknown');
+  log.label('     Server time', serverTimestamp || 'unknown');
+  if (diffColumns.length > 0) {
+    log.dim('     Column changes on server:');
+    for (const { col, serverValue, baselineValue } of diffColumns) {
+      log.dim(`       ${col}:`);
+      if (baselineValue) log.dim(`         was:  ${baselineValue}`);
+      log.dim(`         now:  ${serverValue}`);
+    }
+  }
+}
+
+/**
+ * Find the oldest _LastUpdated date among the baseline entries for the
+ * records about to be pushed. This is used as the UpdatedAfter parameter
+ * to limit the app object response to only recently modified records.
+ *
+ * @param {Array<{ meta: Object }>} records
+ * @param {Object} baseline
+ * @returns {string|null} - ISO date string or null if no baseline dates found
+ */
+function findOldestBaselineDate(records, baseline) {
+  let oldest = null;
+  for (const { meta } of records) {
+    if (!meta.UID || !meta._entity) continue;
+    const entry = findBaselineEntry(baseline, meta._entity, meta.UID);
+    if (!entry?._LastUpdated) continue;
+    const d = new Date(entry._LastUpdated);
+    if (isNaN(d)) continue;
+    if (!oldest || d < oldest) oldest = d;
+  }
+  return oldest ? oldest.toISOString() : null;
+}
+
+/**
+ * Main toe-stepping check. Compares each record being pushed against the
+ * live server state.
+ *
+ * When appShortName is provided, uses a single bulk fetch via
+ *   GET /api/app/object/{appShortName}?UpdatedAfter={date}
+ * which is far more efficient than per-record fetches. Falls back to
+ * per-record GET /api/o/e/{entity}/{uid} when the bulk endpoint is
+ * unavailable or returns no data.
+ *
+ * The bulk response is used ONLY for comparison — it does NOT replace
+ * app.json or app_baseline.json.
+ *
+ * @param {Array<{ meta: Object, metaPath: string }>} records
+ *   Records about to be pushed. Each must have meta.UID, meta._entity.
+ * @param {DboClient} client
+ * @param {Object} baseline - Loaded baseline from .dbo/.app_baseline.json
+ * @param {Object} options  - Commander options (options.yes used for auto-accept)
+ * @param {string} [appShortName] - App short name for bulk fetch (optional)
+ * @returns {Promise<boolean>} - true = proceed, false = user cancelled
+ */
+export async function checkToeStepping(records, client, baseline, options, appShortName) {
+  // Build list of records to check (skip new records without UID)
+  const requests = [];
+  for (const { meta } of records) {
+    if (meta.UID && meta._entity) {
+      requests.push({ entity: meta._entity, uid: meta.UID });
+    }
+  }
+
+  if (requests.length === 0) return true; // nothing to check
+
+  const ora = (await import('ora')).default;
+  const spinner = ora(`Checking ${requests.length} record(s) for server conflicts...`).start();
+
+  // Try bulk fetch via /api/app/object/ with UpdatedAfter when possible
+  let serverRecords;
+  if (appShortName) {
+    const updatedAfter = findOldestBaselineDate(records, baseline);
+    if (updatedAfter) {
+      spinner.text = `Fetching server state (UpdatedAfter: ${updatedAfter})...`;
+      serverRecords = await fetchServerRecordsBatch(client, appShortName, updatedAfter);
+    }
+  }
+
+  // Fall back to per-record fetches if batch returned nothing or wasn't available
+  if (!serverRecords || serverRecords.size === 0) {
+    spinner.text = `Fetching ${requests.length} record(s) from server...`;
+    serverRecords = await fetchServerRecords(client, requests);
+  }
+
+  if (serverRecords.size === 0) {
+    spinner.stop();
+    log.dim('  Toe-stepping: no server records fetched — skipping conflict check');
+    return true; // degrade gracefully
+  }
+
+  spinner.succeed(`Fetched ${serverRecords.size} record(s) from server`);
+
+  const conflicts = [];
+
+  for (const { meta, metaPath } of records) {
+    const uid = meta.UID;
+    const entity = meta._entity;
+    if (!uid || !entity) continue;
+
+    const serverEntry = serverRecords.get(uid);
+    if (!serverEntry) continue; // fetch failed or record not on server — skip
+
+    const baselineEntry = findBaselineEntry(baseline, entity, uid);
+
+    // Compare _LastUpdated: server newer than baseline → someone else changed it
+    const serverTs = serverEntry._LastUpdated;
+    const baselineTs = baselineEntry?._LastUpdated;
+
+    if (!serverTs || !baselineTs) continue; // missing timestamps — skip safely
+
+    // Parse both as dates (ISO 8601 strings or server-format timestamps)
+    const serverDate = new Date(serverTs);
+    const baselineDate = new Date(baselineTs);
+
+    if (isNaN(serverDate) || isNaN(baselineDate)) continue; // unparseable — skip
+    if (serverDate <= baselineDate) continue; // server is same or older — no conflict
+
+    // Conflict detected: server changed since our baseline
+    const metaDir = dirname(metaPath);
+    const label = basename(metaPath, '.metadata.json');
+    const diffColumns = await buildRecordDiff(serverEntry, baselineEntry, meta, metaDir);
+    const serverUser = serverEntry._LastUpdatedUserID || 'unknown';
+
+    displayConflict(label, serverUser, serverTs, diffColumns);
+    conflicts.push({ label, serverUser });
+  }
+
+  if (conflicts.length === 0) return true; // no conflicts
+
+  log.warn('');
+  log.warn(`  ${conflicts.length} record(s) have server changes that would be overwritten.`);
+  log.warn('');
+
+  if (options.yes) {
+    log.dim('  --yes flag: proceeding despite server conflicts');
+    return true;
+  }
+
+  const inquirer = (await import('inquirer')).default;
+  const { action } = await inquirer.prompt([{
+    type: 'list',
+    name: 'action',
+    message: 'Server has newer changes. How would you like to proceed?',
+    choices: [
+      { name: 'Overwrite server changes (push anyway)', value: 'overwrite' },
+      { name: 'Cancel — pull server changes first', value: 'cancel' },
+    ],
+  }]);
+
+  if (action === 'cancel') {
+    log.info('Push cancelled. Run "dbo pull" to fetch server changes first.');
+    return false;
+  }
+
+  log.dim('  Proceeding — local changes will overwrite server state.');
+  return true;
+}

package/src/migrations/001-transaction-key-preset-scope.js

@@ -0,0 +1,35 @@
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+import { log } from '../lib/logger.js';
+
+export const description = 'Restrict TransactionKeyPreset to records with a UID column';
+
+/**
+ * Migration 001 — TransactionKeyPreset scope correction.
+ *
+ * The pushFromMetadata() function now only consults TransactionKeyPreset when
+ * a record carries a UID column (meta.UID is non-null and non-empty).
+ * Records without a UID column (data entities) always use RowID regardless of
+ * the preset. This migration notifies users of the behaviour change.
+ *
+ * @param {object} _options - Command options (unused)
+ */
+export default async function run(_options) {
+  let preset;
+  try {
+    const configPath = join(process.cwd(), '.dbo', 'config.json');
+    const raw = await readFile(configPath, 'utf8');
+    preset = JSON.parse(raw).TransactionKeyPreset;
+  } catch {
+    // Not in a project dir or config unreadable — nothing to notify
+    return;
+  }
+
+  if (!preset) return; // Key not set — no notice needed
+
+  log.plain('');
+  log.dim('  TransactionKeyPreset now applies only to records with a UID column.');
+  log.dim('  Data records without a UID always use RowID (no preset applied).');
+  log.dim(`  Your current preset: ${preset} — no config change required.`);
+  log.plain('');
+}