@dboio/cli 0.8.2 → 0.9.3

package/src/lib/ignore.js

@@ -19,7 +19,9 @@ app.json
 .app.json
 dbo.deploy.json
 
-# Editor / IDE
+# Editor / IDE / OS
+.DS_Store
+Thumbs.db
 .idea/
 .vscode/
 *.codekit3
@@ -39,6 +41,10 @@ node_modules/
 package.json
 package-lock.json
 
+# Local development (not pushed to DBO server)
+src/
+tests/
+
 # Documentation (repo scaffolding)
 SETUP.md
 README.md
@@ -58,7 +64,7 @@ export function getDefaultFileContent() {
 /**
  * Extract just the active pattern lines from DEFAULT_FILE_CONTENT.
  */
-function getDefaultPatternLines() {
+export function getDefaultPatternLines() {
   return DEFAULT_FILE_CONTENT
     .split('\n')
     .filter(l => l && !l.startsWith('#'));

package/src/lib/input-parser.js

@@ -1,9 +1,23 @@
 import { readFile } from 'fs/promises';
 import { log } from './logger.js';
 import { loadUserInfo, loadTicketSuggestionOutput } from './config.js';
-import { buildTicketExpression, setGlobalTicket, setRecordTicket, getGlobalTicket, getRecordTicket } from './ticketing.js';
+import { buildTicketExpression, setGlobalTicket, setRecordTicket, getGlobalTicket, getRecordTicket, setTicketingRequired } from './ticketing.js';
 import { DboClient } from './client.js';
 
+// Session-level cache: once the user resolves a UserID prompt, reuse it for all
+// subsequent records in the same CLI invocation (one process = one push batch).
+let _sessionUserOverride = null;
+
+/** Get cached UserID override (set after first interactive prompt). */
+export function getSessionUserOverride() {
+  return _sessionUserOverride;
+}
+
+/** Reset session caches (for tests). */
+export function resetSessionCache() {
+  _sessionUserOverride = null;
+}
+
 /**
  * Parse DBO input syntax and build form data.
  *
@@ -160,9 +174,16 @@ export async function checkSubmitErrors(result, context = {}) {
     return Object.keys(retryParams).length > 0 ? retryParams : null;
   }
 
+  // Session cache: reuse UserID from a previous prompt in this batch
+  const userResolved = needsUser && _sessionUserOverride;
+  if (userResolved) {
+    retryParams['_OverrideUserID'] = _sessionUserOverride;
+    log.dim(`  Using cached user ID: ${_sessionUserOverride}`);
+  }
+
   const prompts = [];
 
-  if (needsUser) {
+  if (needsUser && !userResolved) {
     log.warn(`This operation requires an authenticated user (${matchedUserPattern}).`);
     log.dim('  Your session may have expired, or you may not be logged in.');
     log.dim('  You can log in with "dbo login" to avoid this prompt in the future.');
@@ -215,7 +236,9 @@ export async function checkSubmitErrors(result, context = {}) {
   const userValue = answers.userValue
     || (answers.userChoice === '_custom' ? answers.customUserValue : answers.userChoice);
   if (userValue) {
-    retryParams['_OverrideUserID'] = userValue.trim();
+    const resolved = userValue.trim();
+    retryParams['_OverrideUserID'] = resolved;
+    _sessionUserOverride = resolved; // cache for remaining records in this batch
   }
 
   return retryParams;
@@ -226,6 +249,9 @@ export async function checkSubmitErrors(result, context = {}) {
  * Prompts the user with 4 recovery options.
  */
 async function handleTicketError(allText, context = {}) {
+  // Mark this app as requiring ticketing (detected on first ticket_error)
+  await setTicketingRequired();
+
   // Try to extract record details from error text
   const entityMatch = allText.match(/entity:(\w+)/);
   const rowIdMatch = allText.match(/RowID:(\d+)/);
@@ -312,8 +338,8 @@ async function handleTicketError(allText, context = {}) {
       name: 'ticketAction',
       message: 'Record update requires a Ticket ID:',
       choices: [
-        { name: 'Apply a Ticket ID to this record and resubmit', value: 'apply_one' },
         { name: 'Apply a Ticket ID to all updates in this transaction, and update my current Ticket ID reference', value: 'apply_all' },
+        { name: 'Apply a Ticket ID to this record only', value: 'apply_one' },
         { name: 'Skip this record update', value: 'skip_one' },
         { name: 'Skip all updates that require a Ticket ID', value: 'skip_all' },
       ],

package/src/lib/metadata-templates.js

@@ -0,0 +1,264 @@
+import { readFile, writeFile } from 'fs/promises';
+import { relative, basename, extname } from 'path';
+import { EXTENSION_DESCRIPTORS_DIR, ENTITY_DIR_NAMES } from './structure.js';
+import inquirer from 'inquirer';
+
+const METADATA_TEMPLATES_FILE = '.dbo/metadata_templates.json';
+
+const STATIC_DIRECTIVE_MAP = {
+  docs: { entity: 'extension', descriptor: 'documentation' },
+};
+
+/**
+ * Convert a name to snake_case.
+ */
+export function toSnakeCase(name) {
+  return name
+    .toLowerCase()
+    .replace(/[\s-]+/g, '_')
+    .replace(/[^a-z0-9_]/g, '');
+}
+
+/**
+ * Resolve entity/descriptor directive from a file path.
+ * Returns { entity, descriptor } or null.
+ */
+export function resolveDirective(filePath) {
+  const rel = relative(process.cwd(), filePath).replace(/\\/g, '/');
+  const parts = rel.split('/');
+  const topDir = parts[0];
+
+  // docs/ prefix → static mapping
+  if (topDir === 'docs') {
+    return { ...STATIC_DIRECTIVE_MAP.docs };
+  }
+
+  // extension/<descriptor>/ — need at least 3 parts (extension/descriptor/file)
+  if (topDir === 'extension') {
+    if (parts.length < 3) return null;
+    const secondDir = parts[1];
+    if (secondDir.startsWith('.')) return null;
+    return { entity: 'extension', descriptor: secondDir };
+  }
+
+  // Other entity-dir types
+  if (ENTITY_DIR_NAMES.has(topDir)) {
+    return { entity: topDir, descriptor: null };
+  }
+
+  return null;
+}
+
+/**
+ * Load metadata templates from .dbo/metadata_templates.json.
+ */
+export async function loadMetadataTemplates() {
+  try {
+    const raw = await readFile(METADATA_TEMPLATES_FILE, 'utf8');
+    try {
+      return JSON.parse(raw);
+    } catch {
+      console.warn('Warning: .dbo/metadata_templates.json is malformed JSON — falling back to generic wizard');
+      return null;
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') return {};
+    return {};
+  }
+}
+
+/**
+ * Save metadata templates to .dbo/metadata_templates.json.
+ */
+export async function saveMetadataTemplates(templates) {
+  await writeFile(METADATA_TEMPLATES_FILE, JSON.stringify(templates, null, 2) + '\n');
+}
+
+/**
+ * Get template cols for a given entity and optional descriptor.
+ * Returns string[] or null.
+ */
+export function getTemplateCols(templates, entity, descriptor) {
+  if (!templates || !templates[entity]) return null;
+  const entry = templates[entity];
+
+  if (descriptor) {
+    // entry must be an object (not array)
+    if (Array.isArray(entry)) return null;
+    return entry[descriptor] ?? null;
+  }
+
+  // no descriptor — entry must be an array
+  if (!Array.isArray(entry)) return null;
+  return entry;
+}
+
+/**
+ * Set template cols for a given entity and optional descriptor.
+ * Mutates templates in place.
+ */
+export function setTemplateCols(templates, entity, descriptor, cols) {
+  if (descriptor) {
+    if (!templates[entity] || Array.isArray(templates[entity])) {
+      templates[entity] = {};
+    }
+    templates[entity][descriptor] = cols;
+  } else {
+    templates[entity] = cols;
+  }
+}
+
+/**
+ * Build cols from a record's keys, excluding _ prefixed and array values.
+ */
+function buildColsFromRecord(record) {
+  return Object.keys(record).filter(key => {
+    if (key.startsWith('_')) return false;
+    if (Array.isArray(record[key])) return false;
+    return true;
+  });
+}
+
+/**
+ * Build baseline minimum viable cols for an entity type.
+ */
+function buildBaselineCols(entity, descriptor) {
+  const cols = ['AppID', 'Name'];
+  if (entity === 'extension') {
+    cols.push('ShortName');
+    if (descriptor) cols.push(`Descriptor=${descriptor}`);
+    cols.push('Active');
+  }
+  return cols;
+}
+
+/**
+ * Resolve template cols via three-level lookup:
+ * 1. metadata_templates.json
+ * 2. appJson sample record
+ * 3. baseline defaults
+ */
+export async function resolveTemplateCols(entity, descriptor, appConfig, appJson) {
+  const templates = await loadMetadataTemplates();
+  if (templates === null) return null;
+
+  // Level 1: existing template
+  const existing = getTemplateCols(templates, entity, descriptor);
+  if (existing) return { cols: existing, templates, isNew: false };
+
+  // Level 2: derive from appJson
+  if (appJson) {
+    let records = null;
+
+    if (entity === 'extension' && descriptor) {
+      // appJson[entity] might be an object keyed by descriptor
+      const entityData = appJson[entity];
+      if (entityData) {
+        if (Array.isArray(entityData)) {
+          records = entityData;
+        } else if (typeof entityData === 'object' && entityData[descriptor]) {
+          const descData = entityData[descriptor];
+          if (Array.isArray(descData)) records = descData;
+        }
+      }
+    } else {
+      const entityData = appJson[entity];
+      if (Array.isArray(entityData)) records = entityData;
+    }
+
+    if (records && records.length > 0) {
+      const cols = buildColsFromRecord(records[0]);
+      setTemplateCols(templates, entity, descriptor, cols);
+      await saveMetadataTemplates(templates);
+      return { cols, templates, isNew: true };
+    }
+  }
+
+  // Level 3: baseline defaults
+  const cols = buildBaselineCols(entity, descriptor);
+  setTemplateCols(templates, entity, descriptor, cols);
+  await saveMetadataTemplates(templates);
+  return { cols, templates, isNew: true };
+}
+
+/**
+ * Assemble metadata object from cols and file info.
+ */
+export function assembleMetadata(cols, filePath, entity, descriptor, appConfig) {
+  const rel = relative(process.cwd(), filePath).replace(/\\/g, '/');
+  const base = basename(filePath, extname(filePath));
+  const fileName = basename(filePath);
+  const isDocsFile = rel.startsWith('docs/');
+  const meta = { _entity: entity };
+  const contentColumns = [];
+
+  for (const col of cols) {
+    if (col.includes('=')) {
+      const eqIdx = col.indexOf('=');
+      const key = col.substring(0, eqIdx);
+      const val = col.substring(eqIdx + 1);
+
+      if (val === '@reference') {
+        const refPath = isDocsFile ? '@/' + rel : '@' + fileName;
+        meta[key] = refPath;
+        contentColumns.push(key);
+      } else {
+        meta[key] = val;
+      }
+    } else if (col === 'AppID') {
+      meta.AppID = appConfig?.AppID ?? '';
+    } else if (col === 'Name') {
+      meta.Name = base;
+    } else if (col === 'ShortName') {
+      meta.ShortName = toSnakeCase(base);
+    } else {
+      meta[col] = '';
+    }
+  }
+
+  let refColMissing = false;
+  if (contentColumns.length === 0) {
+    refColMissing = true;
+  } else {
+    meta._contentColumns = contentColumns;
+  }
+
+  return { meta, contentColumns, refColMissing };
+}
+
+/**
+ * Prompt user to select a reference column from available string cols.
+ */
+export async function promptReferenceColumn(cols, entity, descriptor) {
+  const stringCols = cols.filter(c => !c.includes('=') && c !== 'AppID' && c !== 'Name' && c !== 'ShortName');
+  if (stringCols.length === 0) return null;
+
+  const { selected } = await inquirer.prompt([{
+    type: 'list',
+    name: 'selected',
+    message: `Select the content/reference column for ${entity}${descriptor ? '/' + descriptor : ''}:`,
+    choices: stringCols,
+  }]);
+
+  return selected;
+}
+
+/**
+ * Build a template cols array from a clone record.
+ */
+export function buildTemplateFromCloneRecord(record, contentColsExtracted = []) {
+  const cols = [];
+  for (const key of Object.keys(record)) {
+    if (key.startsWith('_')) continue;
+    if (Array.isArray(record[key])) continue;
+
+    if (contentColsExtracted.includes(key)) {
+      cols.push(key + '=@reference');
+    } else if (key === 'Descriptor' && record[key]) {
+      cols.push('Descriptor=' + record[key]);
+    } else {
+      cols.push(key);
+    }
+  }
+  return cols;
+}

package/src/lib/structure.js

@@ -4,19 +4,22 @@ import { join } from 'path';
 const STRUCTURE_FILE = '.dbo/structure.json';
 
 /** All bin-placed files go under this directory at project root */
-export const BINS_DIR = 'Bins';
+export const BINS_DIR = 'bins';
 
 /** Default top-level directories created at project root during clone */
 export const DEFAULT_PROJECT_DIRS = [
-  BINS_DIR,
-  'Automations',
-  'App Versions',
-  'Documentation',
-  'Sites',
-  'Extensions',
-  'Data Sources',
-  'Groups',
-  'Integrations',
+  'bins',
+  'automation',
+  'app_version',
+  'docs',
+  'site',
+  'extension',
+  'data_source',
+  'group',
+  'integration',
+  'src',
+  'tests',
+  'trash',
 ];
 
 /** Map from physical output table names → documentation/display names */
@@ -35,16 +38,16 @@ export const OUTPUT_HIERARCHY_ENTITIES = [
   'output_value_entity_column_rel',
 ];
 
-/** Map from server entity key → local project directory name */
-export const ENTITY_DIR_MAP = {
-  extension:    'Extensions',
-  app_version:  'App Versions',
-  data_source:  'Data Sources',
-  site:         'Sites',
-  group:        'Groups',
-  integration:  'Integrations',
-  automation:   'Automations',
-};
+/** Entity keys that correspond to project directories (key IS the dir name) */
+export const ENTITY_DIR_NAMES = new Set([
+  'extension',
+  'app_version',
+  'data_source',
+  'site',
+  'group',
+  'integration',
+  'automation',
+]);
 
 /**
  * Build a bin hierarchy from an array of bin objects.
@@ -160,11 +163,12 @@ export function getBinName(binId, structure) {
 
 /**
  * Reverse-lookup: find a bin entry by its directory path.
- * Accepts paths with or without the "Bins/" prefix.
+ * Accepts paths with or without the "bins/" prefix.
  * Returns { binId, name, path, segment, parentBinID, uid, fullPath } or null.
  */
 export function findBinByPath(dirPath, structure) {
-  const normalized = dirPath.replace(/^Bins\//, '').replace(/^\/+|\/+$/g, '');
+  const binsPrefix = BINS_DIR + '/';
+  const normalized = dirPath.replace(new RegExp('^' + binsPrefix), '').replace(/^\/+|\/+$/g, '');
   for (const [binId, entry] of Object.entries(structure)) {
     if (entry.fullPath === normalized) {
       return { binId: Number(binId), ...entry };
@@ -183,13 +187,13 @@ export function findChildBins(binId, structure) {
 // ─── Extension Descriptor Sub-directory Support ───────────────────────────
 
 /** Root for all extension descriptor-grouped sub-directories */
-export const EXTENSION_DESCRIPTORS_DIR = 'Extensions';
+export const EXTENSION_DESCRIPTORS_DIR = 'extension';
 
 /** Extensions that cannot be mapped go here (always created, even if empty) */
-export const EXTENSION_UNSUPPORTED_DIR = 'Extensions/Unsupported';
+export const EXTENSION_UNSUPPORTED_DIR = 'extension/_unsupported';
 
 /** Root-level documentation directory for alternate placement */
-export const DOCUMENTATION_DIR = 'Documentation';
+export const DOCUMENTATION_DIR = 'docs';
 
 /**
  * Build a descriptor→dirName mapping from a flat array of extension records.
@@ -280,7 +284,7 @@ export async function loadDescriptorMapping() {
 
 /**
  * Resolve the sub-directory path for a single extension record.
- * Returns "Extensions/<MappedName>" or "Extensions/Unsupported".
+ * Returns "extension/<MappedName>" or "extension/_unsupported".
  *
  * @param {Object} record   - Extension record with a .Descriptor field
  * @param {Object} mapping  - descriptor key → dir name (from buildDescriptorMapping)

package/src/lib/ticketing.js

@@ -13,7 +13,7 @@ function ticketingPath() {
   return join(dboDir(), TICKETING_FILE);
 }
 
-const DEFAULT_TICKETING = { ticket_id: null, records: [] };
+const DEFAULT_TICKETING = { ticket_id: null, ticketing_required: false, records: [] };
 
 /**
  * Load ticketing.local.json. Returns default structure if missing or corrupted.
@@ -30,6 +30,7 @@ export async function loadTicketing() {
     const data = JSON.parse(trimmed);
     return {
       ticket_id: data.ticket_id || null,
+      ticketing_required: !!data.ticketing_required,
       records: Array.isArray(data.records) ? data.records : [],
     };
   } catch (err) {
@@ -97,6 +98,25 @@ export async function setRecordTicket(uid, rowId, entity, ticketId) {
   await saveTicketing(data);
 }
 
+/**
+ * Mark ticketing as required for this app (detected after first ticket_error).
+ */
+export async function setTicketingRequired() {
+  const data = await loadTicketing();
+  if (!data.ticketing_required) {
+    data.ticketing_required = true;
+    await saveTicketing(data);
+  }
+}
+
+/**
+ * Check if ticketing is required for this app.
+ */
+export async function isTicketingRequired() {
+  const data = await loadTicketing();
+  return !!data.ticketing_required;
+}
+
 /**
  * Clear the global ticket_id (preserves records).
  */
@@ -152,18 +172,66 @@ export async function checkStoredTicket(options, context = '') {
   }
 
   const data = await loadTicketing();
-  if (!data.ticket_id) {
+
+  // No stored ticket and ticketing not known to be required — skip prompt
+  if (!data.ticket_id && !data.ticketing_required) {
     return { useTicket: false, clearTicket: false, cancel: false };
   }
 
-  // Non-interactive mode: auto-use stored ticket
+  // Non-interactive mode: auto-use stored ticket if available
   if (!process.stdin.isTTY) {
-    log.info(`Using stored ticket "${data.ticket_id}" (non-interactive mode)`);
-    return { useTicket: true, clearTicket: false, cancel: false };
+    if (data.ticket_id) {
+      log.info(`Using stored ticket "${data.ticket_id}" (non-interactive mode)`);
+      return { useTicket: true, clearTicket: false, cancel: false };
+    }
+    // ticketing_required but no stored ticket — can't prompt in non-interactive
+    log.warn('This app requires a Ticket ID but none is stored.');
+    log.dim('  Use --ticket <id> or run interactively first to set a ticket.');
+    return { useTicket: false, clearTicket: false, cancel: false };
   }
 
   const suffix = context ? ` (${context})` : '';
   const inquirer = (await import('inquirer')).default;
+
+  // Ticketing required but no stored ticket — prompt for one
+  if (!data.ticket_id && data.ticketing_required) {
+    const { action } = await inquirer.prompt([{
+      type: 'list',
+      name: 'action',
+      message: `This app requires a Ticket ID. Enter one for this submission?${suffix}`,
+      choices: [
+        { name: 'Enter a Ticket ID for this and future submissions', value: 'enter_save' },
+        { name: 'Enter a Ticket ID for this submission only', value: 'enter_once' },
+        { name: 'Continue without a ticket', value: 'skip' },
+        { name: 'Cancel submission', value: 'cancel' },
+      ],
+    }]);
+
+    if (action === 'cancel') {
+      return { useTicket: false, clearTicket: false, cancel: true };
+    }
+    if (action === 'skip') {
+      return { useTicket: false, clearTicket: false, cancel: false };
+    }
+
+    const { ticketInput } = await inquirer.prompt([{
+      type: 'input',
+      name: 'ticketInput',
+      message: 'Ticket ID:',
+    }]);
+    const ticket = ticketInput.trim();
+    if (!ticket) {
+      log.error('  No Ticket ID entered. Submission cancelled.');
+      return { useTicket: false, clearTicket: false, cancel: true };
+    }
+    if (action === 'enter_save') {
+      await saveTicketing({ ...data, ticket_id: ticket });
+      log.dim(`  Stored ticket set to "${ticket}"`);
+    }
+    return { useTicket: true, clearTicket: false, cancel: false, overrideTicket: ticket };
+  }
+
+  // Stored ticket exists — prompt to use, change, or clear
   const { action } = await inquirer.prompt([{
     type: 'list',
     name: 'action',