@dboio/cli 0.9.8 → 0.11.1

package/src/commands/pull.js

@@ -1,114 +1,295 @@
 import { Command } from 'commander';
+import { access } from 'fs/promises';
+import { join } from 'path';
 import chalk from 'chalk';
-import { DboClient } from '../lib/client.js';
-import { loadConfig } from '../lib/config.js';
-import { formatError } from '../lib/formatter.js';
-import { saveToDisk } from '../lib/save-to-disk.js';
+import { performClone, resolveRecordPaths, resolveMediaPaths, resolveEntityDirPaths, resolveEntityFilter, buildOutputFilename } from './clone.js';
+import { loadConfig, loadClonePlacement } from '../lib/config.js';
+import { loadStructureFile, resolveBinPath, BINS_DIR, EXTENSION_DESCRIPTORS_DIR, EXTENSION_UNSUPPORTED_DIR, resolveEntityDirPath } from '../lib/structure.js';
 import { log } from '../lib/logger.js';
-import { renameToUidConvention, hasUidInFilename } from '../lib/filenames.js';
+import { formatError } from '../lib/formatter.js';
+import { getLocalSyncTime, isServerNewer, loadBaselineForComparison } from '../lib/diff.js';
+import { DboClient } from '../lib/client.js';
+import { runPendingMigrations } from '../lib/migrations.js';
 
-function collect(value, previous) {
-  return previous.concat([value]);
+async function fileExists(path) {
+  try { await access(path); return true; } catch { return false; }
 }
 
-export const pullCommand = new Command('pull')
-  .description('Pull records from DBO.io to local files (default entity: content)')
-  .argument('[uid]', 'Record UID (optional)')
-  .option('-e, --entity <uid>', 'Entity to pull from (default: content)', 'content')
-  .option('--filter <expr>', 'Filter expression (repeatable)', collect, [])
-  .option('--maxrows <n>', 'Maximum rows to pull')
-  .option('--sort <expr>', 'Sort expression (repeatable)', collect, [])
-  .option('-v, --verbose', 'Show HTTP request details')
-  .option('--domain <host>', 'Override domain')
-  .action(async (uid, options) => {
-    try {
-      const client = new DboClient({ domain: options.domain, verbose: options.verbose });
-      const entity = options.entity;
-      const params = { '_format': 'json_raw' };
-      if (options.maxrows) params['_maxrows'] = options.maxrows;
-
-      for (const f of options.filter) {
-        const atIdx = f.indexOf('=');
-        if (atIdx !== -1) {
-          const key = f.substring(0, atIdx);
-          const value = f.substring(atIdx + 1);
-          if (key.includes(':')) {
-            const [col, mod] = key.split(':');
-            params[`_filter@${col}:${mod}`] = value;
-          } else {
-            params[`_filter@${key}`] = value;
-          }
-        }
+/**
+ * Fetch the full app JSON from the server (same endpoint as clone).
+ * Distinguishes between authentication failures and genuine "app not found".
+ */
+async function fetchAppJson(config, options) {
+  if (!config.AppShortName) {
+    throw new Error('No AppShortName in config. Run "dbo clone" first to set up the project.');
+  }
+
+  const appShortName = config.AppShortName;
+  const client = new DboClient({ domain: options.domain || config.domain, verbose: options.verbose });
+  const ora = (await import('ora')).default;
+  const spinner = ora(`Fetching app "${appShortName}" from server...`).start();
+
+  let result;
+  try {
+    result = await client.get(`/api/app/object/${appShortName}`);
+  } catch (err) {
+    spinner.fail(`Failed to fetch app "${appShortName}"`);
+    throw err;
+  }
+
+  // Check for authentication / session errors before parsing app data
+  const AUTH_PATTERNS = ['LoggedInUser_UID', 'LoggedInUserID', 'CurrentUserID', 'UserID', 'not authenticated', 'session expired', 'login required'];
+  const messages = result.messages || [];
+  const allMsgText = messages.filter(m => typeof m === 'string').join(' ');
+  const isAuthError = !result.ok && (result.status === 401 || result.status === 403)
+    || (!result.successful && AUTH_PATTERNS.some(p => allMsgText.includes(p)));
+
+  if (isAuthError) {
+    spinner.fail('Session expired or not authenticated');
+    log.warn('Your session appears to have expired.');
+    if (allMsgText) log.dim(`  Server: ${allMsgText.substring(0, 200)}`);
+
+    if (process.stdin.isTTY) {
+      const inquirer = (await import('inquirer')).default;
+      const { action } = await inquirer.prompt([{
+        type: 'list',
+        name: 'action',
+        message: 'How would you like to proceed?',
+        choices: [
+          { name: 'Re-login now (recommended)', value: 'relogin' },
+          { name: 'Abort', value: 'abort' },
+        ],
+      }]);
+
+      if (action === 'relogin') {
+        const { performLogin } = await import('./login.js');
+        await performLogin(options.domain || config.domain);
+        log.info('Retrying app fetch...');
+        return fetchAppJson(config, options);
       }
+    } else {
+      log.dim('  Run "dbo login" to authenticate, then retry.');
+    }
+    throw new Error('Authentication required. Run "dbo login" first.');
+  }
+
+  // Check for non-auth server errors
+  if (!result.ok && result.status >= 500) {
+    spinner.fail(`Server error (HTTP ${result.status})`);
+    if (allMsgText) log.dim(`  Server: ${allMsgText.substring(0, 200)}`);
+    throw new Error(`Server error (HTTP ${result.status}) fetching app "${appShortName}"`);
+  }
+
+  if (!result.successful && allMsgText) {
+    spinner.fail(`Server returned an error`);
+    log.warn(`  ${allMsgText.substring(0, 300)}`);
+    throw new Error(`Server error fetching app "${appShortName}": ${allMsgText.substring(0, 200)}`);
+  }
+
+  const data = result.payload || result.data;
+  let appRecord;
+  if (Array.isArray(data)) {
+    appRecord = data.length > 0 ? data[0] : null;
+  } else if (data?.Rows?.length > 0) {
+    appRecord = data.Rows[0];
+  } else if (data?.rows?.length > 0) {
+    appRecord = data.rows[0];
+  } else if (data && typeof data === 'object' && (data.UID || data.ShortName)) {
+    appRecord = data;
+  } else {
+    appRecord = null;
+  }
+
+  if (!appRecord || !appRecord.children) {
+    spinner.fail(`No app found or invalid response for "${appShortName}"`);
+    throw new Error(`No app found with ShortName "${appShortName}"`);
+  }
+
+  spinner.succeed('Fetched app from server');
+  return appRecord;
+}
+
+/**
+ * Dry-run scan: compare server records against local metadata files.
+ * Reports counts of new, changed, and up-to-date records without writing anything.
+ */
+async function dryRunScan(appJson, options) {
+  const config = await loadConfig();
+  const serverTz = config.ServerTimezone || 'America/Los_Angeles';
+  const configWithTz = { ...config, ServerTimezone: serverTz };
+
+  // Pre-load baseline for fast _LastUpdated string comparison in isServerNewer
+  await loadBaselineForComparison();
+
+  const structure = await loadStructureFile();
+  const saved = await loadClonePlacement();
+  const contentPlacement = saved.contentPlacement || 'bin';
+
+  const entityFilter = resolveEntityFilter(options.entity);
+
+  const counts = { new: 0, changed: 0, upToDate: 0 };
+  const details = { new: [], changed: [] };
+
+  // Scan content records
+  if (!entityFilter || entityFilter.has('content')) {
+    for (const record of (appJson.children.content || [])) {
+      const { metaPath } = resolveRecordPaths('content', record, structure, contentPlacement);
+      await scanRecord(record, metaPath, serverTz, configWithTz, counts, details, 'content');
+    }
+  }
 
-      // Pass sort values through directly — API expects _sort=Column:DIR
-      if (options.sort.length > 0) {
-        params['_sort'] = options.sort;
+  // Scan media records
+  if (!entityFilter || entityFilter.has('media')) {
+    for (const record of (appJson.children.media || [])) {
+      const { metaPath } = resolveMediaPaths(record, structure);
+      await scanRecord(record, metaPath, serverTz, configWithTz, counts, details, 'media');
+    }
+  }
+
+  // Scan entity-dir records (extension, site, data_source, etc.)
+  for (const [entityName, entries] of Object.entries(appJson.children)) {
+    if (['bin', 'content', 'media', 'output', 'output_value', 'output_value_filter', 'output_value_entity_column_rel'].includes(entityName)) continue;
+    if (!Array.isArray(entries) || entries.length === 0) continue;
+    if (entityFilter && !entityFilter.has(entityName)) continue;
+
+    if (entityName === 'extension') {
+      // Extensions use descriptor sub-directories
+      for (const record of entries) {
+        const dirName = resolveExtensionDir(record);
+        const { metaPath } = resolveEntityDirPaths(entityName, record, dirName);
+        await scanRecord(record, metaPath, serverTz, configWithTz, counts, details, entityName);
+      }
+    } else {
+      for (const record of entries) {
+        const { metaPath } = resolveEntityDirPaths(entityName, record, resolveEntityDirPath(entityName));
+        await scanRecord(record, metaPath, serverTz, configWithTz, counts, details, entityName);
       }
+    }
+  }
 
-      let path;
-      if (uid) {
-        path = `/api/output/entity/${entity}/${uid}`;
-      } else {
-        path = `/api/output/entity/${entity}`;
+  // Scan output hierarchy (compound single-file format)
+  if (!entityFilter || entityFilter.has('output')) {
+    const outputEntries = appJson.children.output || [];
+    for (const record of outputEntries) {
+      // Resolve bin directory for this output (same logic as clone)
+      let binDir = BINS_DIR;
+      if (record.BinID && structure[record.BinID]) {
+        binDir = resolveBinPath(record.BinID, structure);
       }
+      const rootBasename = buildOutputFilename('output', record, 'Name');
+      const metaPath = join(binDir, `${rootBasename}.json`);
+      await scanRecord(record, metaPath, serverTz, configWithTz, counts, details, 'output');
+    }
+  }
 
-      const result = await client.get(path, params);
+  // Print summary
+  log.plain('');
+  log.info('Dry-run scan results:');
+  log.plain('');
 
-      log.success(`Pulled from ${chalk.underline(result.url || '')}`);
+  const total = counts.new + counts.changed + counts.upToDate;
 
-      const data = result.payload || result.data;
+  if (counts.new > 0) {
+    log.success(`  ${chalk.green(counts.new)} new record(s)`);
+    for (const d of details.new.slice(0, 10)) {
+      log.dim(`    + [${d.entity}] ${d.name}`);
+    }
+    if (details.new.length > 10) log.dim(`    ... and ${details.new.length - 10} more`);
+  }
 
-      // Validate we got actual JSON data, not raw HTML
-      if (data && data.raw && typeof data.raw === 'string') {
-        log.error(`Received non-JSON response. Check that entity "${entity}" exists.`);
-        log.dim(`  Hint: use --entity (double dash) not -entity`);
-        process.exit(1);
-      }
+  if (counts.changed > 0) {
+    log.warn(`  ${chalk.yellow(counts.changed)} changed record(s)`);
+    for (const d of details.changed.slice(0, 10)) {
+      log.dim(`    ~ [${d.entity}] ${d.name}`);
+    }
+    if (details.changed.length > 10) log.dim(`    ... and ${details.changed.length - 10} more`);
+  }
 
-      const rows = Array.isArray(data) ? data : (data?.Rows || data?.rows || [data]);
+  if (counts.upToDate > 0) {
+    log.dim(`  ${counts.upToDate} up-to-date record(s)`);
+  }
 
-      if (rows.length === 0) {
-        log.warn('No records found.');
-        return;
-      }
+  log.plain('');
+  if (counts.new === 0 && counts.changed === 0) {
+    log.success(`All records up to date (${total} checked)`);
+  } else {
+    log.info(`${total} record(s) checked. Run "dbo pull" (without --dry-run) to apply changes.`);
+  }
+}
 
-      log.info(`${rows.length} record(s) to save`);
+/**
+ * Check a single record against its local metadata file.
+ */
+async function scanRecord(record, metaPath, _serverTz, configWithTz, counts, details, entityName) {
+  const name = record.Name || record.UID || 'unknown';
+  const exists = await fileExists(metaPath);
 
-      const columns = Object.keys(rows[0]);
+  if (!exists) {
+    counts.new++;
+    details.new.push({ entity: entityName, name });
+    return;
+  }
 
-      if (columns.length <= 1) {
-        log.error(`Response has no usable columns. Check the entity name and UID.`);
-        process.exit(1);
-      }
+  const localSyncTime = await getLocalSyncTime(metaPath);
+  if (record._LastUpdated && isServerNewer(localSyncTime, record._LastUpdated, configWithTz, entityName, record.UID)) {
+    counts.changed++;
+    details.changed.push({ entity: entityName, name });
+  } else {
+    counts.upToDate++;
+  }
+}
+
+/**
+ * Resolve the directory for an extension record (descriptor-based sub-dirs).
+ */
+function resolveExtensionDir(record) {
+  const descriptor = record.Descriptor;
+  if (!descriptor) return EXTENSION_UNSUPPORTED_DIR;
+  if (descriptor === 'descriptor_definition') return EXTENSION_DESCRIPTORS_DIR;
+  // Use the descriptor value as sub-directory name
+  return `${EXTENSION_DESCRIPTORS_DIR}/${descriptor}`;
+}
 
-      // Content entity has well-known defaults — skip prompts
-      const isContent = entity === 'content';
+export const pullCommand = new Command('pull')
+  .description('Pull updates from server into existing project')
+  .option('-e, --entity <type>', 'Only pull a specific entity type')
+  .option('-y, --yes', 'Auto-accept all incoming changes')
+  .option('-n, --dry-run', 'Show what would change without writing files')
+  .option('-v, --verbose', 'Show HTTP request details')
+  .option('--domain <host>', 'Override domain')
+  .option('--force', 'Force re-processing, skip change detection')
+  .option('--descriptor-types <bool>', 'Sort extensions into descriptor sub-directories (default: true)', 'true')
+  .option('--documentation-only', 'When used with -e extension, pull only documentation extensions')
+  .option('--no-migrate', 'Skip pending migrations for this invocation')
+  .action(async (options) => {
+    try {
+      await runPendingMigrations(options);
       const config = await loadConfig();
 
-      if (isContent) {
-        await saveToDisk(rows, columns, {
-          entity,
-          saveFilename: columns.includes('Name') ? 'Name' : 'UID',
-          savePath: columns.includes('Path') ? 'Path' : null,
-          saveContent: columns.includes('Content') ? 'Content' : null,
-          saveExtension: columns.includes('Extension') ? 'Extension' : null,
-          nonInteractive: false, // still prompt for overwrites
-          contentFileMap: columns.includes('Content') ? {
-            Content: { suffix: '', extensionSource: columns.includes('Extension') ? 'Extension' : null, extension: 'txt' },
-          } : null,
-          changeDetection: true,
-          config,
-        });
-      } else {
-        // Other entities: interactive prompts for column selection
-        await saveToDisk(rows, columns, {
-          entity,
-          nonInteractive: false,
-          changeDetection: true,
-          config,
-        });
+      if (!config.AppShortName) {
+        log.error('No AppShortName found in .dbo/config.json.');
+        log.dim('  Run "dbo clone" first to set up the project.');
+        process.exit(1);
+      }
+
+      // Dry-run mode: scan and report without writing
+      if (options.dryRun) {
+        const appJson = await fetchAppJson(config, options);
+        await dryRunScan(appJson, options);
+        return;
       }
+
+      // Full pull: delegate to performClone with pullMode
+      await performClone(null, {
+        pullMode: true,
+        yes: options.yes,
+        entity: options.entity,
+        force: options.force,
+        domain: options.domain,
+        verbose: options.verbose,
+        descriptorTypes: options.descriptorTypes,
+        documentationOnly: options.documentationOnly,
+      });
     } catch (err) {
       formatError(err);
       process.exit(1);