@dboio/cli 0.17.0 → 0.19.1

package/src/lib/schema.js

@@ -1,53 +1,83 @@
-import { readFile, writeFile } from 'fs/promises';
-import { DboClient } from './client.js';
+import { readFile } from 'fs/promises';
+import { join, sep } from 'path';
 
-export const SCHEMA_FILE = 'schema.json';
-const SCHEMA_API_PATH = '/api/app/object/_system';
+const SYSTEM_BASELINE_PATH = join('app_dependencies', '_system', '.app', '_system.json');
 
-/** Fetch schema from /api/app/object/_system. Returns parsed JSON or throws. */
-export async function fetchSchema(options = {}) {
-  const client = new DboClient({ domain: options.domain, verbose: options.verbose });
-  const result = await client.get(SCHEMA_API_PATH);
-  if (!result.ok || !result.data) throw new Error(`Schema fetch failed: HTTP ${result.status}`);
-  return result.data;
+/**
+ * Kept for call-site compatibility with commands that pass systemSchemaPath.
+ * Points to the _system dependency baseline location.
+ */
+export const SCHEMA_FILE = SYSTEM_BASELINE_PATH;
+
+/**
+ * Load schema from the _system dependency baseline.
+ * When inside a dependency checkout (app_dependencies/<dep>/), traverses up
+ * to the parent project's _system baseline.
+ * Returns parsed object or null if the dependency hasn't been cloned yet.
+ */
+export async function loadSchema() {
+  // Try local path first (works for top-level projects)
+  try {
+    return JSON.parse(await readFile(SYSTEM_BASELINE_PATH, 'utf8'));
+  } catch { /* not found locally */ }
+
+  // If inside a dependency checkout, try parent project's schema
+  const cwd = process.cwd();
+  const marker = `${sep}app_dependencies${sep}`;
+  const depIdx = cwd.indexOf(marker);
+  if (depIdx >= 0) {
+    const parentRoot = cwd.substring(0, depIdx);
+    const parentSchemaPath = join(parentRoot, SYSTEM_BASELINE_PATH);
+    try {
+      return JSON.parse(await readFile(parentSchemaPath, 'utf8'));
+    } catch { /* parent schema not available either */ }
+  }
+
+  return null;
 }
 
 /**
- * Fetch only the _LastUpdated timestamp from the schema endpoint.
- * Uses the column-filter syntax: /api/app/object/_system[_LastUpdated]
- * Returns ISO date string or null on failure.
+ * saveSchema() is now a no-op — schema is managed by the _system dependency clone.
+ * Kept for call-site compatibility.
  */
-export async function fetchSchemaLastUpdated(options = {}) {
-  const client = new DboClient({ domain: options.domain, verbose: options.verbose });
-  const result = await client.get(`${SCHEMA_API_PATH}[_LastUpdated]`);
-  if (!result.ok || !result.data) return null;
-  return result.data._LastUpdated || null;
+export async function saveSchema(_data) {
+  // no-op
 }
 
 /**
- * Check whether the server schema is newer than the local schema.json.
- * Returns true if the server _LastUpdated is more recent (or local is missing).
+ * Check whether the _system dependency needs refreshing.
+ * Delegates to checkDependencyStaleness('_system').
  */
 export async function isSchemaStale(options = {}) {
-  const local = await loadSchema();
-  if (!local || !local._LastUpdated) return true;
-
-  const serverTs = await fetchSchemaLastUpdated(options);
-  if (!serverTs) return false; // can't determine — assume not stale
-
-  return new Date(serverTs) > new Date(local._LastUpdated);
+  const { checkDependencyStaleness } = await import('./dependencies.js');
+  return checkDependencyStaleness('_system', options);
 }
 
-/** Load schema.json from project root. Returns parsed object or null if missing. */
-export async function loadSchema() {
-  try {
-    return JSON.parse(await readFile(SCHEMA_FILE, 'utf8'));
-  } catch {
-    return null;
+/**
+ * "Fetch" schema by forcing a re-clone of the _system dependency.
+ * When inside a dependency checkout, falls back to direct API fetch
+ * (syncDependencies would skip due to the self-detection guard).
+ * Returns the freshly loaded schema after sync completes.
+ */
+export async function fetchSchema(options = {}) {
+  const cwd = process.cwd();
+  if (cwd.includes(`${sep}app_dependencies${sep}`)) {
+    // Inside a dependency checkout — can't sync _system here.
+    // Try parent's schema first (already cloned by parent).
+    const existing = await loadSchema();
+    if (existing) return existing;
+
+    // Fallback: fetch directly from API (old behavior)
+    const { DboClient } = await import('./client.js');
+    const { loadConfig } = await import('./config.js');
+    const config = await loadConfig();
+    const domain = options.domain || config.domain;
+    const client = new DboClient({ domain, verbose: options.verbose });
+    const resp = await client.get('/api/app/object/_system');
+    return resp?.data || resp;
   }
-}
 
-/** Write schema.json to project root. */
-export async function saveSchema(data) {
-  await writeFile(SCHEMA_FILE, JSON.stringify(data, null, 2) + '\n');
+  const { syncDependencies } = await import('./dependencies.js');
+  await syncDependencies({ ...options, only: ['_system'], force: true });
+  return loadSchema();
 }

package/src/lib/structure.js

@@ -1,7 +1,7 @@
 import { readFile, writeFile, mkdir } from 'fs/promises';
 import { join } from 'path';
 
-const STRUCTURE_FILE = '.dbo/structure.json';
+const STRUCTURE_FILE = '.app/directories.json';
 
 /** All server-managed directories live under this subdirectory */
 export const LIB_DIR = 'lib';
@@ -271,15 +271,15 @@ export async function createDirectories(structure) {
 }
 
 /**
- * Save the bin structure to .dbo/structure.json.
+ * Save the bin structure to .app/directories.json.
  */
 export async function saveStructureFile(structure) {
-  await mkdir('.dbo', { recursive: true });
+  await mkdir('.app', { recursive: true });
   await writeFile(STRUCTURE_FILE, JSON.stringify(structure, null, 2) + '\n');
 }
 
 /**
- * Load bin structure from .dbo/structure.json.
+ * Load bin structure from .app/directories.json.
  */
 export async function loadStructureFile() {
   try {
@@ -389,7 +389,7 @@ export function resolveFieldValue(value) {
 }
 
 /**
- * Persist descriptorMapping and extensionDescriptorDirs into .dbo/structure.json.
+ * Persist descriptorMapping and extensionDescriptorDirs into .app/directories.json.
  * Extends the existing structure object (already contains bin entries).
  *
  * @param {Object} structure  - Current structure from loadStructureFile()
@@ -411,7 +411,7 @@ export async function saveDescriptorMapping(structure, mapping) {
 }
 
 /**
- * Load descriptorMapping from .dbo/structure.json.
+ * Load descriptorMapping from .app/directories.json.
  * Returns {} if not yet persisted.
  */
 export async function loadDescriptorMapping() {

package/src/lib/tagging.js

@@ -149,13 +149,13 @@ async function _findUntrackedFiles(dir, ig, knownMetaPaths) {
   return all.filter(fp => !knownCompanions.has(fp));
 }
 
-// Recursively collect non-metadata, non-hidden, non-.dbo content files
+// Recursively collect non-metadata, non-hidden, non-.app content files
 async function _collectContentFiles(dir, ig) {
   const results = [];
   let entries;
   try { entries = await readdir(dir, { withFileTypes: true }); } catch { return []; }
   for (const entry of entries) {
-    if (entry.name.startsWith('.')) continue; // skip hidden and .dbo
+    if (entry.name.startsWith('.')) continue; // skip hidden and .app
     const fullPath = join(dir, entry.name);
     const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
     if (entry.isDirectory()) {

package/src/lib/ticketing.js

@@ -1,16 +1,12 @@
 import { readFile, writeFile, mkdir } from 'fs/promises';
 import { join } from 'path';
 import { log } from './logger.js';
+import { projectDir } from './config.js';
 
-const DBO_DIR = '.dbo';
 const TICKETING_FILE = 'ticketing.local.json';
 
-function dboDir() {
-  return join(process.cwd(), DBO_DIR);
-}
-
 function ticketingPath() {
-  return join(dboDir(), TICKETING_FILE);
+  return join(projectDir(), TICKETING_FILE);
 }
 
 const DEFAULT_TICKETING = { ticket_id: null, ticketing_required: false, ticket_confirmed: false, records: [] };
@@ -48,7 +44,7 @@ export async function loadTicketing() {
  * Save ticketing.local.json.
  */
 export async function saveTicketing(data) {
-  await mkdir(dboDir(), { recursive: true });
+  await mkdir(projectDir(), { recursive: true });
   await writeFile(ticketingPath(), JSON.stringify(data, null, 2) + '\n');
 }
 

package/src/lib/toe-stepping.js

@@ -87,8 +87,8 @@ export async function fetchServerRecords(client, requests) {
  * makes a single HTTP request and the server only returns records modified
  * after the given date.
  *
- * The response is used ONLY for comparison — it must NOT replace app.json or
- * app_baseline.json.
+ * The response is used ONLY for comparison — it must NOT replace the metadata
+ * or baseline files.
  *
  * @param {DboClient} client
  * @param {string} appShortName - App short name for the /api/app/object/ endpoint
@@ -181,7 +181,7 @@ function decodeServerValue(value) {
  * a newer server edit, and _LastUpdatedUserID identifies who made the change.
  *
  * @param {Object} serverEntry    - Record from per-record server fetch
- * @param {Object} baselineEntry  - Record from .app_baseline.json
+ * @param {Object} baselineEntry  - Record from the baseline file
  * @param {Object} localMeta      - Local .metadata.json object
  * @param {string} metaDir        - Absolute directory of the metadata file
  * @returns {Promise<Array<{ col: string, serverValue: string, localValue: string, baselineValue: string }>>}
@@ -278,12 +278,12 @@ function findOldestBaselineDate(records, baseline) {
  * unavailable or returns no data.
  *
  * The bulk response is used ONLY for comparison — it does NOT replace
- * app.json or app_baseline.json.
+ * the metadata or baseline files.
  *
  * @param {Array<{ meta: Object, metaPath: string }>} records
  *   Records about to be pushed. Each must have meta.UID, meta._entity.
  * @param {DboClient} client
- * @param {Object} baseline - Loaded baseline from .dbo/.app_baseline.json
+ * @param {Object} baseline - Loaded baseline from .app/<shortName>.json (baseline)
  * @param {Object} options  - Commander options (options.yes used for auto-accept)
  * @param {string} [appShortName] - App short name for bulk fetch (optional)
  * @param {string} [serverTz] - Server timezone from config (e.g. "America/Chicago")

package/src/lib/transaction-key.js

@@ -41,6 +41,6 @@ export async function resolveTransactionKey(options = {}) {
   }]);
 
   await saveTransactionKeyPreset(preset);
-  log.dim(`  Saved TransactionKeyPreset: ${preset} to .dbo/config.json`);
+  log.dim(`  Saved TransactionKeyPreset: ${preset} to .app/config.json`);
   return preset;
 }

package/src/migrations/004-rename-output-files.js

@@ -73,10 +73,10 @@ async function rewriteReferences(filePath) {
 
 /**
  * Recursively find directories that contain _output~ files.
- * Skips .dbo/, node_modules/, trash/, .git/.
+ * Skips .app/, node_modules/, trash/, .git/.
  */
 async function findDirsWithOutputFiles(root) {
-  const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+  const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
   const results = [];
 
   async function walk(dir) {

package/src/migrations/005-rename-output-metadata.js

@@ -57,10 +57,10 @@ export default async function run(_options) {
 
 /**
  * Recursively find directories that contain output .json files (with ~ in name).
- * Skips .dbo/, node_modules/, trash/, .git/.
+ * Skips .app/, node_modules/, trash/, .git/.
  */
 async function findDirsWithOutputJsonFiles(root) {
-  const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+  const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
   const results = [];
 
   async function walk(dir) {

package/src/migrations/006-remove-uid-companion-filenames.js

@@ -161,7 +161,7 @@ export default async function run(_options) {
   }
 }
 
-const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
 
 async function findAllMetadataFiles(dir) {
   const results = [];

package/src/migrations/007-natural-entity-companion-filenames.js

@@ -127,7 +127,7 @@ const ENTITY_DIRS = new Set([
   'data_source', 'group', 'site', 'redirect',
 ]);
 
-const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
 
 /**
  * Find all .metadata.json files inside entity directories (lib/<entity>/).

package/src/migrations/008-metadata-uid-in-suffix.js

@@ -50,7 +50,7 @@ export default async function run(_options) {
   }
 }
 
-const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
 
 async function findAllLegacyMetadataFiles(dir) {
   const results = [];

package/src/migrations/009-fix-media-collision-metadata-names.js

@@ -141,7 +141,7 @@ export default async function run(_options) {
   }
 }
 
-const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
 
 async function findAllMetadataFiles(dir) {
   const results = [];

package/src/migrations/010-delete-paren-media-orphans.js

@@ -41,7 +41,7 @@ export default async function run(_options) {
   }
 }
 
-const SKIP = new Set(['.dbo', 'node_modules', 'trash', '.git', '.claude']);
+const SKIP = new Set(['.app', 'node_modules', 'trash', '.git', '.claude', 'app_dependencies']);
 
 async function findParenMediaFiles(dir) {
   const results = [];

package/src/migrations/012-project-dir-restructure.js

@@ -0,0 +1,211 @@
+import { readFile, writeFile, rename, mkdir, rm, readdir, access } from 'fs/promises';
+import { join } from 'path';
+import { homedir } from 'os';
+
+export const description = 'Rename .dbo/ → .app/; rename baseline/structure/metadata files; move dependencies → app_dependencies/; move config.local.json → ~/.dbo/settings.json; remove schema.json';
+
+async function fileExists(p) {
+  try { await access(p); return true; } catch { return false; }
+}
+
+export default async function run() {
+  const cwd = process.cwd();
+  const log = (...args) => console.log(...args);
+
+  const dboDirPath = join(cwd, '.dbo');
+  const appDirPath = join(cwd, '.app');
+
+  // Guard: skip if already on new layout
+  if (!(await fileExists(dboDirPath)) || (await fileExists(appDirPath))) {
+    log('  .dbo/ not found or .app/ already exists — skipping');
+    return;
+  }
+
+  // Step 1: Read AppShortName from .dbo/config.json BEFORE rename
+  let appShortName = null;
+  try {
+    const cfg = JSON.parse(await readFile(join(dboDirPath, 'config.json'), 'utf8'));
+    appShortName = cfg.AppShortName || null;
+  } catch { /* no config — proceed without AppShortName */ }
+
+  const safeName = appShortName
+    ? String(appShortName).replace(/[/\\:*?"<>|]/g, '-')
+    : null;
+
+  // Step 2: Rename .dbo/ → .app/
+  try {
+    await rename(dboDirPath, appDirPath);
+    log(`  Renamed .dbo/ → .app/`);
+  } catch (err) {
+    log(`  (skip) Could not rename .dbo/ → .app/: ${err.message}`);
+    return;
+  }
+
+  // Step 3: Rename .app/.app_baseline.json → .app/<shortName>.json
+  if (safeName) {
+    const oldBaseline = join(appDirPath, '.app_baseline.json');
+    const newBaseline = join(appDirPath, `${safeName}.json`);
+    if ((await fileExists(oldBaseline)) && !(await fileExists(newBaseline))) {
+      try {
+        await rename(oldBaseline, newBaseline);
+        log(`  Renamed .app/.app_baseline.json → .app/${safeName}.json`);
+      } catch (e) { log(`  (skip) baseline rename: ${e.message}`); }
+    }
+  }
+
+  // Step 4: Rename .app/structure.json → .app/directories.json
+  const oldStructure = join(appDirPath, 'structure.json');
+  const newStructure = join(appDirPath, 'directories.json');
+  if ((await fileExists(oldStructure)) && !(await fileExists(newStructure))) {
+    try {
+      await rename(oldStructure, newStructure);
+      log(`  Renamed .app/structure.json → .app/directories.json`);
+    } catch (e) { log(`  (skip) structure rename: ${e.message}`); }
+  }
+
+  // Step 4b: Rename .app/metadata_schema.json → .app/<shortName>.metadata_schema.json
+  if (safeName) {
+    const oldSchema = join(appDirPath, 'metadata_schema.json');
+    const newSchema = join(appDirPath, `${safeName}.metadata_schema.json`);
+    if ((await fileExists(oldSchema)) && !(await fileExists(newSchema))) {
+      try {
+        await rename(oldSchema, newSchema);
+        log(`  Renamed .app/metadata_schema.json → .app/${safeName}.metadata_schema.json`);
+      } catch (e) { log(`  (skip) metadata_schema rename: ${e.message}`); }
+    }
+  }
+
+  // Step 5: Move .app/dependencies/ → app_dependencies/
+  const oldDepsDir = join(appDirPath, 'dependencies');
+  const newDepsRoot = join(cwd, 'app_dependencies');
+  if (await fileExists(oldDepsDir)) {
+    try {
+      await mkdir(newDepsRoot, { recursive: true });
+      const depNames = await readdir(oldDepsDir);
+      for (const name of depNames) {
+        const src = join(oldDepsDir, name);
+        const dest = join(newDepsRoot, name);
+        if (!(await fileExists(dest))) {
+          await rename(src, dest);
+          log(`  Moved app_dependencies/${name}`);
+        }
+      }
+      try { await rm(oldDepsDir, { recursive: true }); } catch { /* non-empty */ }
+    } catch (e) { log(`  (skip) dependencies move: ${e.message}`); }
+  }
+
+  // Step 6: Move root app.json → .app/<shortName>.metadata.json
+  if (safeName) {
+    const rootAppJson = join(cwd, 'app.json');
+    const newMetaPath = join(appDirPath, `${safeName}.metadata.json`);
+    if ((await fileExists(rootAppJson)) && !(await fileExists(newMetaPath))) {
+      try {
+        await rename(rootAppJson, newMetaPath);
+        log(`  Moved app.json → .app/${safeName}.metadata.json`);
+      } catch (e) { log(`  (skip) app.json move: ${e.message}`); }
+    }
+  }
+
+  // Step 7: Remove root schema.json
+  const rootSchema = join(cwd, 'schema.json');
+  if (await fileExists(rootSchema)) {
+    try {
+      await rm(rootSchema);
+      log(`  Removed schema.json (schema now sourced from app_dependencies/_system/.app/_system.json)`);
+    } catch (e) { log(`  (skip) schema.json removal: ${e.message}`); }
+  }
+
+  // Step 8: Move .app/config.local.json → ~/.dbo/settings.json
+  const oldLocal = join(appDirPath, 'config.local.json');
+  const globalSettingsDir = join(homedir(), '.dbo');
+  const globalSettings = join(globalSettingsDir, 'settings.json');
+  if (await fileExists(oldLocal)) {
+    try {
+      await mkdir(globalSettingsDir, { recursive: true });
+      let oldData = {};
+      try { oldData = JSON.parse(await readFile(oldLocal, 'utf8')); } catch { /* ignore */ }
+
+      let newData = {};
+      try { newData = JSON.parse(await readFile(globalSettings, 'utf8')); } catch { /* new file */ }
+
+      if (oldData.plugins) {
+        if (!newData.plugins) newData.plugins = {};
+        Object.assign(newData.plugins, oldData.plugins);
+      }
+
+      if (Array.isArray(oldData._completedMigrations) && oldData._completedMigrations.length > 0) {
+        if (!newData._completedMigrations) newData._completedMigrations = {};
+        const existing = new Set(Array.isArray(newData._completedMigrations[cwd]) ? newData._completedMigrations[cwd] : []);
+        for (const id of oldData._completedMigrations) existing.add(id);
+        newData._completedMigrations[cwd] = [...existing].sort();
+      }
+
+      await writeFile(globalSettings, JSON.stringify(newData, null, 2) + '\n');
+      try { await rm(oldLocal); } catch { /* leave it */ }
+      log(`  Merged .app/config.local.json → ~/.dbo/settings.json`);
+    } catch (e) { log(`  (skip) config.local.json merge: ${e.message}`); }
+  }
+
+  // Step 9: Update .gitignore
+  const gitignorePath = join(cwd, '.gitignore');
+  try {
+    let content = '';
+    try { content = await readFile(gitignorePath, 'utf8'); } catch { /* no .gitignore */ }
+
+    const replacements = [
+      ['.dbo/credentials.json', '.app/credentials.json'],
+      ['.dbo/cookies.txt', '.app/cookies.txt'],
+      ['.dbo/config.local.json', null],
+      ['.dbo/.app_baseline.json', null],
+      ['.dbo/ticketing.local.json', '.app/ticketing.local.json'],
+      ['.dbo/scripts.local.json', '.app/scripts.local.json'],
+      ['.dbo/errors.log', '.app/errors.log'],
+      ['.dbo/dependencies/', null],
+      ['schema.json', null],
+    ];
+
+    let changed = false;
+    let lines = content.split('\n');
+
+    for (const [oldPat, newPat] of replacements) {
+      const idx = lines.findIndex(l => l.trim() === oldPat);
+      if (idx >= 0) {
+        if (newPat) { lines[idx] = newPat; }
+        else lines.splice(idx, 1);
+        changed = true;
+      } else if (newPat && !content.includes(newPat)) {
+        lines.push(newPat);
+        changed = true;
+      }
+    }
+
+    if (!content.includes('app_dependencies/')) {
+      lines.push('app_dependencies/');
+      changed = true;
+    }
+
+    if (changed) {
+      await writeFile(gitignorePath, lines.join('\n'));
+      log(`  Updated .gitignore with new .app/ paths`);
+    }
+  } catch (e) { log(`  (skip) .gitignore update: ${e.message}`); }
+
+  // Step 10: Update .dboignore
+  const dboignorePath = join(cwd, '.dboignore');
+  try {
+    let content = await readFile(dboignorePath, 'utf8');
+    const updated = content
+      .replace(/^\.dbo\/$/m, '.app/')
+      .replace(/^\.dbo\/dependencies\/$/m, '')
+      .replace(/^app\.json$/m, '');
+    if (!updated.includes('app_dependencies/')) {
+      const finalContent = updated.replace(/^\.app\/$/m, '.app/\napp_dependencies/');
+      await writeFile(dboignorePath, finalContent);
+    } else {
+      await writeFile(dboignorePath, updated);
+    }
+    log(`  Updated .dboignore`);
+  } catch { /* no .dboignore */ }
+
+  log(`  Migration 012 complete: .dbo/ → .app/`);
+}