@dboio/cli 0.19.7 → 0.20.0

package/src/lib/insert.js

@@ -1,4 +1,4 @@
-import { readFile, readdir, writeFile, mkdir, unlink } from 'fs/promises';
+import { readFile, readdir, writeFile, mkdir } from 'fs/promises';
 import { join, dirname, basename, extname, relative } from 'path';
 import { DboClient } from './client.js';
 import { buildInputBody, checkSubmitErrors } from './input-parser.js';
@@ -7,7 +7,7 @@ import { log } from './logger.js';
 import { shouldSkipColumn } from './columns.js';
 import { loadAppConfig, loadAppJsonBaseline, saveAppJsonBaseline, loadExtensionDocumentationMDPlacement, loadDescriptorFilenamePreference, loadConfig } from './config.js';
 import { loadMetadataSchema, saveMetadataSchema } from './metadata-schema.js';
-import { hasUidInFilename, buildMetaFilename, isMetadataFile } from './filenames.js';
+import { isMetadataFile } from './filenames.js';
 import { setFileTimestamps } from './timestamps.js';
 import { checkStoredTicket, clearGlobalTicket } from './ticketing.js';
 import { checkModifyKey, isModifyKeyError, handleModifyKeyError } from './modify-key.js';
@@ -366,51 +366,35 @@ export async function submitAdd(meta, metaPath, filePath, client, options) {
     }
 
     if (returnedUID) {
-      // Check if UID is already embedded in the filename (idempotency guard)
-      const currentMetaBase = basename(metaPath);
-      if (hasUidInFilename(currentMetaBase, returnedUID)) {
-        meta.UID = returnedUID;
-        await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
-        log.success(`UID already in filename: ${returnedUID}`);
-      } else {
-        // Companion file NOT renamed — only update UID in metadata and rename metadata file
-        const metaDir = dirname(metaPath);
-        const metaBase = basename(metaPath, '.metadata.json');
-        const newMetaPath = join(metaDir, buildMetaFilename(metaBase, returnedUID));
-
-        meta.UID = returnedUID;
-
-        await writeFile(newMetaPath, JSON.stringify(meta, null, 2) + '\n');
-        if (metaPath !== newMetaPath) {
-          try { await unlink(metaPath); } catch { /* old file already gone */ }
-        }
-        log.dim(`  Renamed metadata: ${basename(metaPath)} → ${basename(newMetaPath)}`);
-        log.success(`UID assigned: ${returnedUID}`);
-
-        // Restore timestamps for metadata and companion files
-        const config = await loadConfig();
-        const serverTz = config.ServerTimezone;
-        if (serverTz && returnedLastUpdated) {
-          try {
-            await setFileTimestamps(newMetaPath, returnedLastUpdated, returnedLastUpdated, serverTz);
-            for (const col of (meta._companionReferenceColumns || meta._contentColumns || [])) {
-              const ref = meta[col];
-              if (ref && String(ref).startsWith('@')) {
-                const fp = join(metaDir, String(ref).substring(1));
-                await upsertDeployEntry(fp, returnedUID, entity, col);
-                try { await setFileTimestamps(fp, returnedLastUpdated, returnedLastUpdated, serverTz); } catch {}
-              }
-            }
-            if (meta._mediaFile && String(meta._mediaFile).startsWith('@')) {
-              const mediaFp = join(metaDir, String(meta._mediaFile).substring(1));
-              await upsertDeployEntry(mediaFp, returnedUID, entity, 'File');
+      // UID lives in the JSON content — write it in-place, no filename rename needed
+      const metaDir = dirname(metaPath);
+      meta.UID = returnedUID;
+      await writeFile(metaPath, JSON.stringify(meta, null, 2) + '\n');
+      log.success(`UID assigned: ${returnedUID}`);
+
+      // Restore timestamps for metadata and companion files
+      const config = await loadConfig();
+      const serverTz = config.ServerTimezone;
+      if (serverTz && returnedLastUpdated) {
+        try {
+          await setFileTimestamps(metaPath, returnedLastUpdated, returnedLastUpdated, serverTz);
+          for (const col of (meta._companionReferenceColumns || meta._contentColumns || [])) {
+            const ref = meta[col];
+            if (ref && String(ref).startsWith('@')) {
+              const fp = join(metaDir, String(ref).substring(1));
+              await upsertDeployEntry(fp, returnedUID, entity, col);
+              try { await setFileTimestamps(fp, returnedLastUpdated, returnedLastUpdated, serverTz); } catch {}
             }
-          } catch { /* non-critical */ }
-        }
-
-        // Update .app_baseline.json so subsequent pull/diff recognize this record
-        await updateBaselineWithNewRecord(meta, returnedUID, returnedLastUpdated);
+          }
+          if (meta._mediaFile && String(meta._mediaFile).startsWith('@')) {
+            const mediaFp = join(metaDir, String(meta._mediaFile).substring(1));
+            await upsertDeployEntry(mediaFp, returnedUID, entity, 'File');
+          }
+        } catch { /* non-critical */ }
       }
+
+      // Update .app_baseline.json so subsequent pull/diff recognize this record
+      await updateBaselineWithNewRecord(meta, returnedUID, returnedLastUpdated);
       log.dim(`  Run "dbo pull -e ${entity} ${returnedUID}" to populate all columns`);
     }
   }

package/src/lib/structure.js

@@ -16,7 +16,7 @@ export const BINS_DIR = 'lib/bins';
 export const SCAFFOLD_DIRS = [
   'lib/bins',
   'lib/app_version',
-  'lib/entity',
+  'lib/data_source',
   'lib/extension',
   'lib/security',
   'src',
@@ -32,7 +32,6 @@ export const SCAFFOLD_DIRS = [
  */
 export const ON_DEMAND_ENTITY_DIRS = new Set([
   'automation',
-  'data_source',
   'entity_column',
   'entity_column_value',
   'group',
@@ -48,7 +47,6 @@ export const ON_DEMAND_ENTITY_DIRS = new Set([
  */
 export const DEFAULT_PROJECT_DIRS = [
   ...SCAFFOLD_DIRS,
-  'lib/data_source',
   'lib/group',
   'lib/redirect',
   'lib/site',
@@ -87,16 +85,26 @@ export const ENTITY_DIR_NAMES = new Set([
   'site',
 ]);
 
+/**
+ * Entity types that are co-located in another entity's directory.
+ * Maps entity name → directory name (both live under lib/<dirName>/).
+ */
+const ENTITY_DIR_REMAP = {
+  entity: 'data_source',
+};
+
 /**
  * Resolve the local directory path for an entity-dir type.
- * Always returns "lib/<entityName>" (e.g. "lib/extension", "lib/site").
+ * Most entities map to "lib/<entityName>"; remapped entities (e.g. "entity"
+ * → "lib/data_source") are defined in ENTITY_DIR_REMAP.
  * Use this instead of bare entity name concatenation everywhere.
  *
  * @param {string} entityName - Entity key from ENTITY_DIR_NAMES (e.g. "extension")
  * @returns {string} - Path relative to project root (e.g. "lib/extension")
  */
 export function resolveEntityDirPath(entityName) {
-  return `${LIB_DIR}/${entityName}`;
+  const dirName = ENTITY_DIR_REMAP[entityName] ?? entityName;
+  return `${LIB_DIR}/${dirName}`;
 }
 
 // ─── Core Asset Entity Classification ─────────────────────────────────────
@@ -421,7 +429,9 @@ export async function loadDescriptorMapping() {
 
 /**
  * Resolve the sub-directory path for a single extension record.
- * Returns "extension/<MappedName>" or "extension/_unsupported".
+ * Returns "lib/extension/<MappedName>" for mapped descriptors,
+ * "lib/extension/<descriptor>" for unmapped descriptors with a value,
+ * or "lib/extension" for records with no descriptor.
  *
  * @param {Object} record   - Extension record with a .Descriptor field
  * @param {Object} mapping  - descriptor key → dir name (from buildDescriptorMapping)
@@ -429,8 +439,13 @@ export async function loadDescriptorMapping() {
  */
 export function resolveExtensionSubDir(record, mapping) {
   const descriptor = record.Descriptor;
-  if (!descriptor || !mapping[descriptor]) {
+  if (!descriptor) {
     return EXTENSION_DESCRIPTORS_DIR;
   }
-  return `${EXTENSION_DESCRIPTORS_DIR}/${mapping[descriptor]}`;
+  if (mapping[descriptor]) {
+    return `${EXTENSION_DESCRIPTORS_DIR}/${mapping[descriptor]}`;
+  }
+  // Unmapped descriptor (no descriptor_definition record): use the raw descriptor
+  // value as the directory name — matches the directory created by buildDescriptorPrePass.
+  return `${EXTENSION_DESCRIPTORS_DIR}/${descriptor}`;
 }

package/src/lib/ticketing.js

@@ -283,13 +283,14 @@ export async function checkStoredTicket(options, context = '') {
 }
 
 /**
- * Apply a stored ticket to submission data expressions if no --ticket flag is set.
- * Checks per-record ticket first, then global ticket.
+ * Look up and return the active stored ticket for a submission, if no --ticket flag is set.
+ * Checks per-record ticket first, then global ticket. Callers pass the returned value
+ * to _OverrideTicketID in extraParams.
  *
- * @param {string[]} dataExprs - The data expressions array (mutated in place)
- * @param {string} entity - Entity name
- * @param {string|number} rowId - Row ID or UID used in the submission
- * @param {string} uid - Record UID for per-record lookup
+ * @param {string[]} dataExprs - Unused; kept for backwards compatibility
+ * @param {string} entity - Unused; kept for backwards compatibility
+ * @param {string|number} rowId - Unused; kept for backwards compatibility
+ * @param {string} uid - Record UID for per-record ticket lookup
  * @param {Object} options - Command options
  * @param {string|null} [sessionOverride] - One-time ticket override from pre-flight prompt
  */
@@ -301,8 +302,8 @@ export async function applyStoredTicketToSubmission(dataExprs, entity, rowId, ui
   const ticketToUse = sessionOverride || recordTicket || globalTicket;
 
   if (ticketToUse) {
-    const ticketExpr = buildTicketExpression(entity, rowId, ticketToUse);
-    dataExprs.push(ticketExpr);
+    // Don't add _LastUpdatedTicketID to dataExprs — callers set _OverrideTicketID in extraParams,
+    // which is sufficient and avoids server-side RowUID lookup failures on some entities (e.g. extension).
     log.dim(`  Applying ticket: ${ticketToUse}`);
     return ticketToUse;
   }

package/src/migrations/008-metadata-uid-in-suffix.js

@@ -1,7 +1,7 @@
 import { readdir, rename, access } from 'fs/promises';
 import { join, basename, dirname } from 'path';
 import { log } from '../lib/logger.js';
-import { detectLegacyTildeMetadata, buildMetaFilename } from '../lib/filenames.js';
+import { detectLegacyTildeMetadata } from '../lib/filenames.js';
 
 export const description = 'Rename metadata files from name~uid.metadata.json to name.metadata~uid.json';
 
@@ -32,7 +32,9 @@ export default async function run(_options) {
     if (!parsed) continue;
 
     const { naturalBase, uid } = parsed;
-    const newFilename = buildMetaFilename(naturalBase, uid);
+    // Migration 008 target is the intermediate name.metadata~uid.json format;
+    // migration 013 will later remove the uid suffix entirely.
+    const newFilename = `${naturalBase}.metadata~${uid}.json`;
     const newPath = join(dir, newFilename);
 
     // Skip if target already exists (avoid overwrite)

package/src/migrations/009-fix-media-collision-metadata-names.js

@@ -1,7 +1,13 @@
 import { readdir, readFile, rename, unlink, access } from 'fs/promises';
 import { join, basename, dirname, extname } from 'path';
 import { log } from '../lib/logger.js';
-import { parseMetaFilename, buildMetaFilename } from '../lib/filenames.js';
+import { parseMetaFilename } from '../lib/filenames.js';
+
+// Build the intermediate name.metadata~uid.json format this migration targets.
+// (Migration 013 will later strip the uid from filenames entirely.)
+function buildLegacySuffixFilename(naturalBase, uid) {
+  return `${naturalBase}.metadata~${uid}.json`;
+}
 
 export const description = 'Fix media collision suffix: rename (media) → _media and fix mismatched metadata filenames';
 
@@ -87,7 +93,7 @@ export default async function run(_options) {
       // Rename metadata file itself if it contains (media)
       if (parsed.naturalBase.includes('(media)')) {
         const newBase = parsed.naturalBase.replace('(media)', '_media');
-        const newMetaFilename = buildMetaFilename(newBase, parsed.uid);
+        const newMetaFilename = buildLegacySuffixFilename(newBase, parsed.uid);
         const newMetaPath = join(dirname(metaPath), newMetaFilename);
         try { await access(newMetaPath); } catch {
           await rename(metaPath, newMetaPath);
@@ -112,7 +118,7 @@ export default async function run(_options) {
       // Already correct
       if (currentParsed.naturalBase === refFilename) continue;
 
-      const correctFilename = buildMetaFilename(refFilename, currentParsed.uid);
+      const correctFilename = buildLegacySuffixFilename(refFilename, currentParsed.uid);
       const correctPath = join(dirname(metaPath), correctFilename);
 
       // If correct metadata already exists, this one is an orphan

package/src/migrations/013-remove-uid-from-meta-filenames.js

@@ -0,0 +1,117 @@
+import { readFile, writeFile, rename, readdir, access } from 'fs/promises';
+import { join, basename, dirname } from 'path';
+
+export const description = 'Rename metadata files from name.metadata~uid.json to name.metadata.json';
+
+/**
+ * Migration 013 — Remove UID from metadata filenames.
+ *
+ * Old format: colors.metadata~abc123.json
+ * New format: colors.metadata.json
+ *
+ * The UID is already stored inside the JSON as the "UID" field — no information is lost.
+ *
+ * Collision resolution uses entity priority:
+ *   content > output > everything else
+ *
+ * Within the same entity type, the first record (alphabetically by old filename) wins
+ * the unsuffixed slot; subsequent ones get -1, -2, etc.
+ *
+ * Does NOT rename companion files (they use natural names already).
+ */
+export default async function run(_options) {
+  const cwd = process.cwd();
+  let totalRenamed = 0;
+
+  const legacyFiles = await findLegacySuffixMetadataFiles(cwd);
+  if (legacyFiles.length === 0) return;
+
+  // Group by directory so we can resolve collisions per-dir
+  const byDir = new Map();
+  for (const filePath of legacyFiles) {
+    const dir = dirname(filePath);
+    if (!byDir.has(dir)) byDir.set(dir, []);
+    byDir.get(dir).push(filePath);
+  }
+
+  for (const [dir, files] of byDir) {
+    // Read entity type from each file to apply priority ordering
+    const withMeta = [];
+    for (const filePath of files) {
+      let entity = 'other';
+      let uid = null;
+      try {
+        const content = JSON.parse(await readFile(filePath, 'utf8'));
+        entity = content._entity || 'other';
+        uid = content.UID || null;
+      } catch { /* use defaults */ }
+
+      // Parse naturalBase from legacy filename: name.metadata~uid.json
+      const filename = basename(filePath);
+      const m = filename.match(/^(.+)\.metadata~([a-z0-9_]+)\.json$/i);
+      const naturalBase = m ? m[1] : filename.replace(/\.metadata~[^.]+\.json$/i, '');
+
+      withMeta.push({ filePath, naturalBase, entity, uid });
+    }
+
+    // Sort by entity priority: content first, output second, then others (alphabetically within tier)
+    const PRIORITY = { content: 0, output: 1 };
+    withMeta.sort((a, b) => {
+      const pa = PRIORITY[a.entity] ?? 2;
+      const pb = PRIORITY[b.entity] ?? 2;
+      if (pa !== pb) return pa - pb;
+      return a.naturalBase.localeCompare(b.naturalBase);
+    });
+
+    // Assign new filenames with collision resolution
+    const usedBases = new Map(); // naturalBase (lowercase) → count of times used
+
+    for (const { filePath, naturalBase, entity } of withMeta) {
+      const baseKey = naturalBase.toLowerCase();
+      const count = usedBases.get(baseKey) || 0;
+      usedBases.set(baseKey, count + 1);
+
+      const newFilename = count === 0
+        ? `${naturalBase}.metadata.json`
+        : `${naturalBase}-${count}.metadata.json`;
+
+      const newPath = join(dir, newFilename);
+
+      // Skip if target already exists (safe guard against re-running migration)
+      try { await access(newPath); continue; } catch { /* doesn't exist — safe to rename */ }
+
+      try {
+        await rename(filePath, newPath);
+        if (newFilename !== basename(filePath)) {
+          console.log(`  [${entity}] ${basename(filePath)} → ${newFilename}`);
+          totalRenamed++;
+        }
+      } catch (err) {
+        console.warn(`  (skip) Could not rename ${basename(filePath)}: ${err.message}`);
+      }
+    }
+  }
+
+  if (totalRenamed > 0) {
+    console.log(`  Renamed ${totalRenamed} metadata file(s) — UID now stored in JSON only`);
+  }
+}
+
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
+
+async function findLegacySuffixMetadataFiles(dir) {
+  const results = [];
+  try {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (SKIP.has(entry.name)) continue;
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        results.push(...await findLegacySuffixMetadataFiles(full));
+      } else if (/\.metadata~[a-z0-9_]+\.json$/i.test(entry.name)) {
+        results.push(full);
+      }
+    }
+  } catch { /* skip unreadable dirs */ }
+  return results;
+}

package/src/migrations/014-entity-dir-to-data-source.js

@@ -0,0 +1,68 @@
+import { readdir, rename, mkdir, rmdir, access } from 'fs/promises';
+import { join } from 'path';
+import { log } from '../lib/logger.js';
+
+export const description = 'Move lib/entity/ files into lib/data_source/';
+
+/**
+ * Migration 014 — Relocate entity records from lib/entity/ into lib/data_source/.
+ *
+ * Entity (table-definition) records are now co-located with data source records
+ * under lib/data_source/. Files keep their _entity: "entity" metadata value —
+ * only their directory changes.
+ *
+ * If lib/data_source/ already contains a file with the same name, the source
+ * file is left in place and a warning is emitted; no data is overwritten.
+ */
+export default async function run() {
+  const cwd = process.cwd();
+  const srcDir = join(cwd, 'lib', 'entity');
+
+  // Nothing to do if lib/entity/ doesn't exist
+  try {
+    await access(srcDir);
+  } catch {
+    return;
+  }
+
+  const entries = await readdir(srcDir, { withFileTypes: true });
+  if (entries.length === 0) {
+    // Empty directory — just remove it
+    try { await rmdir(srcDir); } catch { /* ignore */ }
+    return;
+  }
+
+  const destDir = join(cwd, 'lib', 'data_source');
+  await mkdir(destDir, { recursive: true });
+
+  let movedCount = 0;
+  let skippedCount = 0;
+
+  for (const entry of entries) {
+    const srcPath = join(srcDir, entry.name);
+    const destPath = join(destDir, entry.name);
+
+    // Check for collision
+    try {
+      await access(destPath);
+      log.warn(`  Migration 014: skipped ${entry.name} — already exists in lib/data_source/`);
+      skippedCount++;
+      continue;
+    } catch { /* dest absent — safe to move */ }
+
+    await rename(srcPath, destPath);
+    movedCount++;
+  }
+
+  // Remove lib/entity/ if now empty
+  try {
+    const remaining = await readdir(srcDir);
+    if (remaining.length === 0) {
+      await rmdir(srcDir);
+    }
+  } catch { /* ignore */ }
+
+  if (movedCount > 0) {
+    log.success(`  Migration 014: moved ${movedCount} file(s) from lib/entity/ → lib/data_source/${skippedCount > 0 ? ` (${skippedCount} skipped — collision)` : ''}`);
+  }
+}