@dboio/cli 0.7.2 → 0.8.2

package/README.md

@@ -98,6 +98,9 @@ dbo --version
 # 1. Initialize configuration for the current directory
 dbo init --domain my-domain.com
 
+# 1b. Combined with cloning an app to local project bin
+dbo init --domain my-domain.com --app myapp --clone
+
 # 2. Authenticate
 dbo login
 
@@ -144,6 +147,7 @@ All configuration is **directory-scoped**. Each project folder maintains its own
   "AppUID": "abc123",
   "AppName": "My App",
   "AppShortName": "myapp",
+  "cloneSource": "default",
   "ContentPlacement": "bin",
   "MediaPlacement": "fullpath"
 }
@@ -159,6 +163,7 @@ All configuration is **directory-scoped**. Each project folder maintains its own
 | `AppModifyKey` | string | ModifyKey for locked/production apps (set by `dbo clone`, used for submission guards) |
 | `TransactionKeyPreset` | `RowUID` \| `RowID` | Row key type for auto-assembled expressions (set during `dbo init`/`dbo clone`, default `RowUID`) |
 | `TicketSuggestionOutput` | string | Output UID for fetching ticket suggestions during error recovery (set during `dbo init`, default `ojaie9t3o0kfvliahnuuda`) |
+| `cloneSource` | `"default"` \| file path \| URL | Where the last `dbo clone` fetched app JSON from. `"default"` = server fetch via `AppShortName`; any other value = the explicit local file path or URL used. Set automatically after each successful clone. |
 | `ContentPlacement` | `bin` \| `path` \| `ask` | Where to place content files during clone |
 | `MediaPlacement` | `bin` \| `fullpath` \| `ask` | Where to place media files during clone |
 | `<Entity>FilenameCol` | column name | Filename column for entity-dir records (e.g., `ExtensionFilenameCol`) |
@@ -174,6 +179,59 @@ These are set interactively on first clone and saved for future use. Pre-set the
 
 The `_domain` field in `app.json` stores the project's reference domain (set during `dbo clone`). This is committed to git and used for domain-change detection when running `dbo init --force` or `dbo clone --domain`. It provides a stable cross-user baseline — all collaborators share the same reference domain.
 
+### `.dboignore`
+
+A gitignore-style file in the project root that controls which files and directories are excluded from CLI operations. Created automatically by `dbo init` with sensible defaults.
+
+**Used by:**
+- `dbo init` — scaffold empty-check (determines if directory is "effectively empty")
+- `dbo add .` — directory scanning (which files/dirs to skip); also checked in single-file mode (`dbo add file.html`)
+- `dbo push` — metadata file discovery: skips matching `.metadata.json` / `_output~*.json` files AND any record whose companion content file (`@reference`) matches an ignore pattern
+
+**Bypass:** Use `dbo input -d '...'` to submit expressions for a file that would otherwise be ignored — `dbo input` never does file discovery so `.dboignore` does not apply.
+
+**Pattern examples:**
+
+```gitignore
+# Ignore all SQL companion files (output records with CustomSQL)
+**/*.CustomSQL.sql
+
+# Ignore a specific record (by metadata file path)
+Bins/app/my-draft-page.metadata.json
+
+# Ignore an entire directory
+Bins/staging/
+
+# Ignore all content in Bins/ (still push entity-dir records like Extensions/)
+Bins/
+```
+
+**Syntax:** Same as `.gitignore` — glob patterns, `#` comments, blank lines, negation with `!`, directory-only patterns with trailing `/`.
+
+**Default patterns:**
+
+```gitignore
+.dbo/              # DBO internal
+*.dboio.json       # Export files
+app.json           # Clone output
+.app.json          # Clone baseline
+dbo.deploy.json    # Deployment manifest
+.git/              # Version control
+.gitignore
+node_modules/      # Node
+package.json
+package-lock.json
+.claude/           # AI / tooling
+.mcp.json
+.idea/             # Editor / IDE
+.vscode/
+*.codekit3
+README.md          # Repo scaffolding
+SETUP.md
+```
+
+Edit `.dboignore` to add or remove patterns. Committed to git and shared across users.
+
 ### Legacy migration
 
 If your project uses the older `.domain`, `.username`, `.password`, `.cookies` files, `dbo init` will detect them and offer to migrate automatically.
@@ -210,11 +268,12 @@ dbo init --scaffold --yes                             # scaffold dirs non-intera
 | `--domain <host>` | DBO instance domain |
 | `--username <user>` | DBO username (stored for login default) |
 | `--force` | Overwrite existing configuration. Triggers a domain-change confirmation prompt when the new domain differs from the project reference domain |
-| `--app <shortName>` | App short name (triggers clone after init) |
+| `--app <shortName>` | App short name (triggers clone after init). Prompts for password and authenticates automatically before fetching app data from the server |
 | `--clone` | Clone the app after initialization |
 | `-g, --global` | Install Claude commands globally (`~/.claude/commands/`) |
 | `--local` | Install Claude commands to project (`.claude/commands/`) |
 | `--scaffold` | Pre-create standard project directories (`App Versions`, `Automations`, `Bins`, `Data Sources`, `Documentation`, `Extensions`, `Groups`, `Integrations`, `Sites`) |
+| `--dboignore` | Create `.dboignore` with default patterns (use with `--force` to overwrite existing) |
 | `-y, --yes` | Skip all interactive prompts (legacy migration, Claude Code setup) |
 | `--non-interactive` | Alias for `--yes` |
 
@@ -236,14 +295,24 @@ dbo clone --app myapp
 
 # Combined with init
 dbo init --domain my-domain.com --app myapp --clone
+
+# Extension descriptor sub-directory handling
+dbo clone -e extension                            # Clone all extensions (sorted by descriptor type)
+dbo clone -e extension --documentation-only       # Clone only documentation extensions
+dbo clone -e extension --documentation-only --force  # Reset documentation preferences and re-clone
+dbo clone -e extension --descriptor-types false   # Clone extensions flat (no descriptor sorting)
 ```
 
 | Flag | Description |
 |------|-------------|
-| `<source>` | Local JSON file path (optional positional argument) |
+| `<source>` | Local file path or URL to load app JSON from (optional positional argument) |
 | `--app <name>` | App short name to fetch from server |
+| `-e, --entity <type>` | Only clone a specific entity type (e.g. `output`, `content`, `media`, `extension`) |
+| `--documentation-only` | When used with `-e extension`, clone only documentation extensions |
+| `--descriptor-types <bool>` | Sort extensions into descriptor sub-directories (default: `true`). Set to `false` to use flat `Extensions/` layout |
 | `--domain <host>` | Override domain. Triggers a domain-change confirmation prompt when it differs from the project reference domain |
-| `-y, --yes` | Auto-accept all prompts |
+| `--force` | Skip source mismatch confirmation and change detection; re-processes all files |
+| `-y, --yes` | Auto-accept all prompts (also skips source mismatch confirmation) |
 | `-v, --verbose` | Show HTTP request details |
 
 #### Domain change detection
@@ -254,17 +323,41 @@ The project's reference domain is stored in `app.json._domain` (committed to git
 
 #### What clone does
 
-1. **Loads app JSON** — from a local file, server API, or interactive prompt
-2. **Updates `.dbo/config.json`** — saves `AppID`, `AppUID`, `AppName`, `AppShortName`, and `AppModifyKey` (if the app is locked)
+1. **Loads app JSON** — from a local file, server API, or interactive prompt. A spinner shows progress while fetching from the server (responses can be slow as the JSON is assembled on demand)
+2. **Updates `.dbo/config.json`** — saves `AppID`, `AppUID`, `AppName`, `AppShortName`, `AppModifyKey` (if the app is locked), and `cloneSource` (the source used for this clone)
 3. **Updates `package.json`** — populates `name`, `productName`, `description`, `homepage`, and `deploy` script
 4. **Creates directories** — processes `children.bin` to build the directory hierarchy based on `ParentBinID` relationships
 5. **Saves `.dbo/structure.json`** — maps BinIDs to directory paths for file placement
-6. **Writes content files** — decodes base64 content, creates `*.metadata.json` + content files in the correct bin directory
+6. **Writes content files** — decodes base64 content, creates `*.metadata.json` + content files in the correct bin directory. When a record's `Extension` field is empty, prompts you to choose from `css`, `js`, `html`, `xml`, `txt`, `md`, `cs`, `json`, `sql` (or skip to keep no extension). The chosen extension is saved back into the `metadata.json` `Extension` field
 7. **Downloads media files** — fetches binary files (images, CSS, fonts) from the server via `/api/media/{uid}` and saves with metadata
 8. **Processes entity-dir records** — entities matching project directories (`extension`, `app_version`, `data_source`, `site`, `group`, `integration`, `automation`) are saved as `.metadata.json` files in their corresponding directory (e.g., `Extensions/`, `Data Sources/`)
 9. **Processes other entities** — remaining entities with a `BinID` are placed in the corresponding bin directory
 10. **Saves `app.json`** — clone of the original JSON with processed entries replaced by `@path/to/*.metadata.json` references
 
+#### Clone source tracking
+
+After every successful clone, the source is persisted as `cloneSource` in `.dbo/config.json`:
+
+- `"default"` — app JSON was fetched from the server using `AppShortName` (the normal flow)
+- A file path or URL — set when an explicit `<source>` argument was provided (e.g. `dbo clone /path/to/export.json`)
+
+On subsequent clones, if you provide an explicit source that **differs** from the stored `cloneSource`, the CLI warns and asks for confirmation before proceeding:
+
+```
+  ⚠  This project was previously cloned from: default
+     Requested source:                        /path/to/export.json
+? Clone from the new source anyway? This will update the stored clone source. (y/N)
+```
+
+Use `--force` or `-y` to skip the prompt and override without being asked.
+
+If the initial clone attempt fails (network error, file not found, empty response), the CLI prompts for a fallback source to retry with instead of aborting:
+
+```
+  ⚠  Source did not return expected results: No app found with ShortName "myapp"
+? Enter another local file path or URL to retry (or leave empty to abort):
+```
+
 #### Change detection on re-clone
 
 When cloning an app that was already cloned locally, the CLI detects existing files and compares modification times against the server's `_LastUpdated`. You'll be prompted to overwrite, compare differences, or skip — same as `dbo pull`. Use `-y` to auto-accept all changes.
@@ -312,7 +405,7 @@ Entity types that correspond to project directories (`extension`, `app_version`,
 
 | Entity Key | Directory |
 |------------|-----------|
-| `extension` | `Extensions/` |
+| `extension` | `Extensions/<Descriptor>/` (see below) |
 | `app_version` | `App Versions/` |
 | `data_source` | `Data Sources/` |
 | `site` | `Sites/` |
@@ -326,6 +419,44 @@ If any columns contain base64-encoded content, the CLI prompts to extract them a
 
 Use `-y` to skip prompts (uses `Name` column, no content extraction).
 
+#### Extension descriptor sub-directories
+
+Extension records are organized into sub-directories under `Extensions/` based on their `Descriptor` column value. A special extension type called `descriptor_definition` (itself an extension with `Descriptor === "descriptor_definition"`) provides the mapping from descriptor key (`String1`) to display/directory name (`Name`).
+
+During clone, the CLI:
+
+1. **Pre-scans** all extension records for `descriptor_definition` entries and builds a `String1 → Name` mapping
+2. **Creates sub-directories** under `Extensions/` for each mapped descriptor (e.g., `Extensions/Documentation/`, `Extensions/Includes/`)
+3. **Always creates** `Extensions/Unsupported/` — extensions with an unmapped or null `Descriptor` are placed here
+4. **Prompts per descriptor** — filename column and content extraction prompts fire once per unique `Descriptor` value (not once for all extensions)
+5. **Persists the mapping** in `.dbo/structure.json` under `descriptorMapping`
+
+**Config keys** (saved per descriptor in `config.json`):
+
+| Key | Example | Purpose |
+|-----|---------|---------|
+| `Extension_<descriptor>_FilenameCol` | `Extension_documentation_FilenameCol` | Filename column for that descriptor type |
+| `Extension_<descriptor>_ContentExtractions` | `Extension_documentation_ContentExtractions` | Content extraction preferences per column |
+| `ExtensionDocumentationMDPlacement` | `"inline"` or `"root"` | Where to place documentation MD files |
+
+**Documentation alternate placement**: When extracting MD content from `documentation` descriptor extensions, the CLI offers a choice:
+
+- **Root placement** (`/Documentation/<filename>.md`) — recommended for easy access; creates `@/Documentation/<filename>.md` references in metadata
+- **Inline placement** (`Extensions/Documentation/<filename>.md`) — keeps files alongside metadata
+
+Root-placed files use absolute-from-root `@/` references (e.g., `@/Documentation/my-doc.md`) so `dbo push` can locate them regardless of where the metadata lives.
+
+**Entity filter support**:
+
+```bash
+dbo clone -e extension --documentation-only         # Clone only documentation extensions
+dbo clone -e extension --documentation-only --force  # Reset documentation preferences and re-clone
+dbo clone -e extension                               # Clone all extensions (all descriptor types)
+dbo clone -e extension --descriptor-types false       # Flat layout (no descriptor sorting)
+```
+
+**`dbo add` auto-inference**: Running `dbo add Documentation/my-doc.md` when `ExtensionDocumentationMDPlacement` is `"root"` auto-creates a companion `.metadata.json` in `Extensions/Documentation/` with the correct `@/` reference.
+
 #### Output structure
 
 ```
@@ -344,9 +475,15 @@ project/
       CurrentTicketID.metadata.json
     ticket_test/             # ← another bin directory
       ...
-  Extensions/                # ← entity-dir records
-    MyExtension.metadata.json
-    MyExtension.CustomCSS.css    # ← extracted content column
+  Extensions/                # ← extension records organized by descriptor
+    Documentation/           # ← descriptor sub-directory (from descriptor_definition)
+      MyDoc.uid1.metadata.json
+      MyDoc.uid1.String10.md     # ← extracted content column
+    Includes/
+      Header.uid2.metadata.json
+    Unsupported/             # ← always created (unmapped/null descriptors)
+  Documentation/             # ← root-placed documentation MD files (optional)
+    MyDoc.md
   Data Sources/
     MySQL-Primary.metadata.json
   Sites/
@@ -896,6 +1033,8 @@ After accepting changes, file modification times are synced to the server's `_La
 
 Push local files back to DBO.io using metadata from a previous pull. This is the counterpart to `dbo content pull` and `dbo output --save`.
 
+Files and records matching `.dboignore` patterns are skipped — both by metadata file path (e.g. `*.metadata.json`) and by companion content file path (e.g. a record whose `@Content` points to an ignored `.sql` file). To push an ignored file directly, use `dbo input -d '...'` with an explicit expression.
+
 #### Round-trip workflow
 
 ```bash
@@ -1069,6 +1208,8 @@ The next `dbo push` processes all pending deletions before pushing file changes.
 
 Add a new file to DBO.io by creating a server record. Similar to `git add`, this registers a local file with the server.
 
+Files matching `.dboignore` patterns are skipped — both in directory-scan mode (`dbo add .`) and single-file mode (`dbo add file.html`). Use `dbo input` to create a record for an ignored file directly.
+
 #### Single file
 
 ```bash
@@ -1357,7 +1498,8 @@ Smart behavior:
 - Compares file hashes — unchanged files are skipped
 - Adds project-scoped plugins to `.gitignore` (source of truth is `plugins/claude/` at repo root)
 - Per-plugin scope (project or global) is stored in `.dbo/config.local.json` and remembered for future upgrades
-- On first install of a plugin without a stored preference, prompts for scope
+- On re-install (e.g. after `npm install` or `npm link`), scope is inferred automatically from the global plugin registry (`~/.claude/plugins/installed_plugins.json`) or existing project installation — no prompt is shown
+- On first install with no prior preference and no existing installation, prompts for scope
 - `--global` / `--local` flags override stored preferences and update them
 
 ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dboio/cli",
-  "version": "0.7.2",
+  "version": "0.8.2",
   "description": "CLI for the DBO.io framework",
   "type": "module",
   "bin": {
@@ -21,8 +21,9 @@
     "test": "node --test src/**/*.test.js tests/**/*.test.js"
   },
   "dependencies": {
-    "commander": "^12.1.0",
     "chalk": "^5.3.0",
+    "commander": "^12.1.0",
+    "ignore": "^7.0.5",
     "inquirer": "^9.3.0"
   },
   "keywords": [

package/src/commands/add.js

@@ -1,17 +1,15 @@
 import { Command } from 'commander';
-import { readFile, readdir, stat, writeFile } from 'fs/promises';
+import { readFile, readdir, stat, writeFile, mkdir } from 'fs/promises';
 import { join, dirname, basename, extname, relative } from 'path';
 import { DboClient } from '../lib/client.js';
 import { buildInputBody, checkSubmitErrors } from '../lib/input-parser.js';
 import { formatResponse, formatError } from '../lib/formatter.js';
 import { log } from '../lib/logger.js';
 import { shouldSkipColumn } from '../lib/columns.js';
-import { loadAppConfig } from '../lib/config.js';
+import { loadAppConfig, loadExtensionDocumentationMDPlacement, loadDescriptorFilenamePreference } from '../lib/config.js';
 import { checkStoredTicket, clearGlobalTicket } from '../lib/ticketing.js';
 import { checkModifyKey, isModifyKeyError, handleModifyKeyError } from '../lib/modify-key.js';
-
-// Directories and patterns to skip when scanning with `dbo add .`
-const IGNORE_DIRS = new Set(['.dbo', '.git', 'node_modules', '.svn', '.hg']);
+import { loadIgnore } from '../lib/ignore.js';
 
 export const addCommand = new Command('add')
   .description('Add a new file to DBO.io (creates record on server)')
@@ -57,7 +55,32 @@ export const addCommand = new Command('add')
  * Add a single file to DBO.io.
  * Returns the defaults used (entity, contentColumn) for batch reuse.
  */
+/**
+ * Detect whether a file lives in the project-root Documentation/ directory
+ * AND the ExtensionDocumentationMDPlacement preference is 'root'.
+ * Returns detection info object or null if not applicable.
+ *
+ * @param {string} filePath - Path to the file being added
+ */
+async function detectDocumentationFile(filePath) {
+  const placement = await loadExtensionDocumentationMDPlacement();
+  if (placement !== 'root') return null;
+
+  const rel = relative(process.cwd(), filePath).replace(/\\/g, '/');
+  if (!rel.startsWith('Documentation/')) return null;
+
+  return { entity: 'extension', descriptor: 'documentation' };
+}
+
 async function addSingleFile(filePath, client, options, batchDefaults) {
+  // Check .dboignore before doing any processing
+  const ig = await loadIgnore();
+  const relPath = relative(process.cwd(), filePath).replace(/\\/g, '/');
+  if (ig.ignores(relPath)) {
+    log.dim(`Skipped (dboignored): ${relPath}`);
+    return null;
+  }
+
   const dir = dirname(filePath);
   const ext = extname(filePath);
   const base = basename(filePath, ext);
@@ -87,6 +110,33 @@ async function addSingleFile(filePath, client, options, batchDefaults) {
     return await submitAdd(meta, metaPath, filePath, client, options);
   }
 
+  // Step 3b: Auto-infer entity/descriptor for Documentation/ files when root placement is configured
+  const docInfo = await detectDocumentationFile(filePath);
+  if (docInfo) {
+    const filenameCol = await loadDescriptorFilenamePreference('documentation') || 'Name';
+    const docBase = basename(filePath, extname(filePath));
+    const companionDir = join(process.cwd(), 'Extensions', 'Documentation');
+    await mkdir(companionDir, { recursive: true });
+
+    const docMeta = {
+      _entity: 'extension',
+      Descriptor: 'documentation',
+      [filenameCol]: docBase,
+      Name: docBase,
+    };
+
+    // Find the content column name from saved preferences
+    const contentCol = filenameCol === 'Name' ? 'String10' : 'String10';
+    const relPath = relative(process.cwd(), filePath).replace(/\\/g, '/');
+    docMeta[contentCol] = `@/${relPath}`;
+    docMeta._contentColumns = [contentCol];
+
+    const docMetaFile = join(companionDir, `${docBase}.metadata.json`);
+    await writeFile(docMetaFile, JSON.stringify(docMeta, null, 2) + '\n');
+    log.success(`Created companion metadata at ${docMetaFile}`);
+    return await submitAdd(docMeta, docMetaFile, filePath, client, options);
+  }
+
   // Step 4: No usable metadata — interactive wizard
   const inquirer = (await import('inquirer')).default;
 
@@ -379,23 +429,26 @@ async function addDirectory(dirPath, client, options) {
  *   - It has no companion .metadata.json, OR
  *   - Its .metadata.json exists but has no _CreatedOn (never been on server)
  */
-async function findUnaddedFiles(dir) {
+async function findUnaddedFiles(dir, ig) {
+  if (!ig) ig = await loadIgnore();
+
   const results = [];
   const entries = await readdir(dir, { withFileTypes: true });
 
   for (const entry of entries) {
     const fullPath = join(dir, entry.name);
+    const relPath = relative(process.cwd(), fullPath).replace(/\\/g, '/');
 
     if (entry.isDirectory()) {
-      if (IGNORE_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
-      results.push(...await findUnaddedFiles(fullPath));
+      if (ig.ignores(relPath + '/') || ig.ignores(relPath)) continue;
+      results.push(...await findUnaddedFiles(fullPath, ig));
       continue;
     }
 
-    // Skip metadata files themselves
+    // Skip metadata files themselves (structural, not user-configurable)
     if (entry.name.endsWith('.metadata.json')) continue;
-    // Skip dotfiles
-    if (entry.name.startsWith('.')) continue;
+    // Skip ignored files
+    if (ig.ignores(relPath)) continue;
 
     // Check for companion metadata
     const ext = extname(entry.name);