@cardstack/boxel-cli 0.1.0 → 0.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cardstack/boxel-cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "license": "MIT",
   "description": "CLI tools for Boxel workspace management",
   "main": "./dist/index.js",
@@ -57,6 +57,8 @@
   },
   "scripts": {
     "build": "pnpm clean && NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/build.ts",
+    "build:plugin": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/build-plugin.ts",
+    "build:skills": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/build-skills.ts",
     "clean": "rm -rf dist/*",
     "start": "NODE_NO_WARNINGS=1 ts-node --transpileOnly src/index.ts",
     "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"",
@@ -72,8 +74,8 @@
     "version:patch": "npm version patch",
     "version:minor": "npm version minor",
     "version:major": "npm version major",
-    "publish:npm": "npm publish",
-    "publish:dry": "npm publish --dry-run"
+    "publish:npm": "pnpm publish --no-git-checks",
+    "publish:dry": "pnpm publish --dry-run --no-git-checks"
   },
   "publishConfig": {
     "access": "public",

package/src/build-program.ts

@@ -0,0 +1,91 @@
+import { Command } from 'commander';
+import { profileCommand } from './commands/profile';
+import { registerReadTranspiledCommand } from './commands/read-transpiled';
+import { registerRealmCommand } from './commands/realm/index';
+import { registerFileCommand } from './commands/file/index';
+import { registerRunCommand } from './commands/run-command';
+import { registerSearchCommand } from './commands/search';
+import { setQuiet } from './lib/cli-log';
+
+/**
+ * Construct the boxel CLI program with every command registered. Pure builder
+ * — does not call `program.parse()` and has no side effects on argv. Both the
+ * runtime entry point (`src/index.ts`) and the plugin generator
+ * (`scripts/build-plugin.ts`) call this so the Commander tree is one source of
+ * truth.
+ */
+export function buildBoxelProgram(version: string): Command {
+  const program = new Command();
+
+  program
+    .name('boxel')
+    .description('CLI tools for Boxel workspace management')
+    .version(version)
+    .option(
+      '-q, --quiet',
+      'Suppress informational progress logs (info/log/debug). Errors and warnings, plus command result payloads (JSON, file contents), are still emitted. Use this when invoking the CLI from automation (e.g. the software factory test harness) to keep stdout focused on the result.',
+    )
+    .hook('preAction', (thisCommand) => {
+      let opts = thisCommand.optsWithGlobals?.() ?? thisCommand.opts();
+      if (opts.quiet) {
+        setQuiet(true);
+      }
+    });
+
+  program
+    .command('profile')
+    .description('Manage saved profiles for different users/environments')
+    .argument('[subcommand]', 'list | add | switch | remove | migrate')
+    .argument('[arg]', 'Profile ID (for switch/remove)')
+    .option('-u, --user <matrixId>', 'Matrix user ID (e.g., @user:boxel.ai)')
+    .option('-p, --password <password>', 'Password (for add command)')
+    .option('-n, --name <displayName>', 'Display name (for add command)')
+    .option(
+      '-m, --matrix-url <url>',
+      'Matrix server URL (for add command with non-standard domains)',
+    )
+    .option(
+      '-r, --realm-server-url <url>',
+      'Realm server URL (for add command with non-standard domains)',
+    )
+    .addHelpText(
+      'after',
+      `
+Environment variables (for 'add'):
+  BOXEL_PASSWORD       Password; preferred over -p to avoid shell history.
+  BOXEL_ENVIRONMENT    An env-mode slug (e.g. a branch name), interpreted
+                       like scripts/env-slug.sh: URLs are derived as
+                       http://matrix.<slug>.localhost and
+                       http://realm-server.<slug>.localhost/. Overridden
+                       by --matrix-url / --realm-server-url if provided.`,
+    )
+    .action(
+      async (
+        subcommand?: string,
+        arg?: string,
+        options?: {
+          user?: string;
+          password?: string;
+          name?: string;
+          matrixUrl?: string;
+          realmServerUrl?: string;
+        },
+      ) => {
+        if (options?.password) {
+          console.warn(
+            'Warning: Supplying a password via -p/--password may expose it in shell history and process listings. ' +
+              'For non-interactive usage, prefer the BOXEL_PASSWORD environment variable or use "boxel profile add" interactively.',
+          );
+        }
+        await profileCommand(subcommand, arg, options);
+      },
+    );
+
+  registerFileCommand(program);
+  registerRealmCommand(program);
+  registerRunCommand(program);
+  registerSearchCommand(program);
+  registerReadTranspiledCommand(program);
+
+  return program;
+}

package/src/commands/realm/push.ts

@@ -25,6 +25,11 @@ interface PushOptions extends SyncOptions {
   force?: boolean;
 }
 
+// Fresh realms always include these server-managed cards even when the local
+// workspace has never pulled them. Treat them as realm artifacts, not user
+// drift, so `push --delete` only removes genuine remote-only user files.
+const REMOTE_DELETE_EXCLUSIONS = new Set(['index.json', 'realm.json']);
+
 class RealmPusher extends RealmSyncBase {
   hasError = false;
 
@@ -225,10 +230,16 @@ class RealmPusher extends RealmSyncBase {
 
     if (this.pushOptions.deleteRemote) {
       const filesToDelete = new Set(initialRemoteFiles.keys());
+      const skippedDeleteArtifacts: string[] = [];
 
       for (const relativePath of filesToDelete) {
         if (isProtectedFile(relativePath)) {
           filesToDelete.delete(relativePath);
+          continue;
+        }
+        if (REMOTE_DELETE_EXCLUSIONS.has(relativePath)) {
+          filesToDelete.delete(relativePath);
+          skippedDeleteArtifacts.push(relativePath);
         }
       }
 
@@ -236,21 +247,26 @@ class RealmPusher extends RealmSyncBase {
         filesToDelete.delete(relativePath);
       }
 
-      if (filesToDelete.size > 0) {
+      if (skippedDeleteArtifacts.length > 0) {
         console.log(
-          `Deleting ${filesToDelete.size} remote files that don't exist locally`,
+          `Skipping ${skippedDeleteArtifacts.length} realm-managed remote artifact(s): ${skippedDeleteArtifacts.join(', ')}`,
         );
+      }
 
-        await Promise.all(
-          Array.from(filesToDelete).map(async (relativePath) => {
-            try {
-              await this.deleteFile(relativePath);
-            } catch (error) {
-              this.hasError = true;
-              console.error(`Error deleting ${relativePath}:`, error);
-            }
-          }),
+      if (filesToDelete.size > 0) {
+        const deletePlan = Array.from(filesToDelete).sort();
+        console.log(
+          `Deleting ${deletePlan.length} remote files that don't exist locally: ${deletePlan.join(', ')}`,
         );
+
+        for (const relativePath of deletePlan) {
+          try {
+            await this.deleteFile(relativePath);
+          } catch (error) {
+            this.hasError = true;
+            console.error(`Error deleting ${relativePath}:`, error);
+          }
+        }
       }
     }
 

package/src/commands/realm/sync.ts

@@ -489,6 +489,12 @@ export interface SyncCommandOptions {
   preferNewest?: boolean;
   delete?: boolean;
   dryRun?: boolean;
+  /**
+   * Append `?waitForIndex=true` to the `_atomic` upload so the
+   * realm-server returns only after the indexer has processed the
+   * batch. See `SyncOptions.waitForIndex` for the rationale.
+   */
+  waitForIndex?: boolean;
   profileManager?: ProfileManager;
   /**
    * Pre-resolved realm secret seed for administrative access. When set, the
@@ -629,6 +635,7 @@ export async function sync(
         preferNewest: options.preferNewest,
         deleteSync: options.delete,
         dryRun: options.dryRun,
+        waitForIndex: options.waitForIndex,
       },
       authenticator,
     );

package/src/index.ts

@@ -1,44 +1,19 @@
 import 'dotenv/config';
-import { Command } from 'commander';
 import { readFileSync } from 'fs';
 import { resolve } from 'path';
-import { profileCommand } from './commands/profile';
-import { registerReadTranspiledCommand } from './commands/read-transpiled';
-import { registerRealmCommand } from './commands/realm/index';
-import { registerFileCommand } from './commands/file/index';
-import { registerRunCommand } from './commands/run-command';
-import { registerSearchCommand } from './commands/search';
+import { buildBoxelProgram } from './build-program';
 import { setQuiet } from './lib/cli-log';
 
 const pkg = JSON.parse(
   readFileSync(resolve(__dirname, '../package.json'), 'utf-8'),
 );
 
-const program = new Command();
-
 // `--quiet` is implemented by intercepting `console.log/info/debug`.
 // New commands: write decorative output (status, confirmations, colored
 // lines) with `console.log` — it's silenced for free under `--quiet`.
 // For programmatic output (`--json` payloads, raw file bytes), use
 // `cliLog.output(...)`. Full guidance: see `lib/cli-log.ts`.
-program
-  .name('boxel')
-  .description('CLI tools for Boxel workspace management')
-  .version(pkg.version)
-  .option(
-    '-q, --quiet',
-    'Suppress informational progress logs (info/log/debug). Errors and warnings, plus command result payloads (JSON, file contents), are still emitted. Use this when invoking the CLI from automation (e.g. the software factory test harness) to keep stdout focused on the result.',
-  )
-  // Toggle quiet mode as soon as the global option is parsed, so that any
-  // module-level setup happening inside command actions sees the right
-  // state. Commander invokes this hook before any subcommand action.
-  .hook('preAction', (thisCommand) => {
-    let opts = thisCommand.optsWithGlobals?.() ?? thisCommand.opts();
-    if (opts.quiet) {
-      setQuiet(true);
-    }
-  });
-
+//
 // Belt-and-suspenders: also flip quiet mode based on a raw scan of argv,
 // so any code that runs between Commander's option parsing and the
 // `preAction` hook sees the right state. We scan for the long form only;
@@ -47,59 +22,4 @@ if (process.argv.includes('--quiet')) {
   setQuiet(true);
 }
 
-program
-  .command('profile')
-  .description('Manage saved profiles for different users/environments')
-  .argument('[subcommand]', 'list | add | switch | remove | migrate')
-  .argument('[arg]', 'Profile ID (for switch/remove)')
-  .option('-u, --user <matrixId>', 'Matrix user ID (e.g., @user:boxel.ai)')
-  .option('-p, --password <password>', 'Password (for add command)')
-  .option('-n, --name <displayName>', 'Display name (for add command)')
-  .option(
-    '-m, --matrix-url <url>',
-    'Matrix server URL (for add command with non-standard domains)',
-  )
-  .option(
-    '-r, --realm-server-url <url>',
-    'Realm server URL (for add command with non-standard domains)',
-  )
-  .addHelpText(
-    'after',
-    `
-Environment variables (for 'add'):
-  BOXEL_PASSWORD       Password; preferred over -p to avoid shell history.
-  BOXEL_ENVIRONMENT    An env-mode slug (e.g. a branch name), interpreted
-                       like scripts/env-slug.sh: URLs are derived as
-                       http://matrix.<slug>.localhost and
-                       http://realm-server.<slug>.localhost/. Overridden
-                       by --matrix-url / --realm-server-url if provided.`,
-  )
-  .action(
-    async (
-      subcommand?: string,
-      arg?: string,
-      options?: {
-        user?: string;
-        password?: string;
-        name?: string;
-        matrixUrl?: string;
-        realmServerUrl?: string;
-      },
-    ) => {
-      if (options?.password) {
-        console.warn(
-          'Warning: Supplying a password via -p/--password may expose it in shell history and process listings. ' +
-            'For non-interactive usage, prefer the BOXEL_PASSWORD environment variable or use "boxel profile add" interactively.',
-        );
-      }
-      await profileCommand(subcommand, arg, options);
-    },
-  );
-
-registerFileCommand(program);
-registerRealmCommand(program);
-registerRunCommand(program);
-registerSearchCommand(program);
-registerReadTranspiledCommand(program);
-
-program.parse();
+buildBoxelProgram(pkg.version).parse();

package/src/lib/boxel-cli-client.ts

@@ -90,6 +90,16 @@ export interface SyncOptions {
   delete?: boolean;
   /** Preview without making changes. */
   dryRun?: boolean;
+  /**
+   * Block on the realm-server until uploaded cards have been indexed,
+   * not just durably written. Appends `?waitForIndex=true` to the
+   * `_atomic` POST. Trades upload latency for read-after-write
+   * consistency — useful when the next step queries the realm's index
+   * (search / list) and can't tolerate the indexer lag introduced by
+   * CS-11003 PR 2's deferred `+source` POST. Off by default; flip on
+   * for hand-off boundaries like the factory's post-seed sync.
+   */
+  waitForIndex?: boolean;
 }
 
 export type { DeleteResult };
@@ -467,6 +477,7 @@ export class BoxelCLIClient {
       preferNewest: options?.preferNewest,
       delete: options?.delete,
       dryRun: options?.dryRun,
+      waitForIndex: options?.waitForIndex,
       profileManager: this.pm,
     });
   }

package/src/lib/realm-sync-base.ts

@@ -9,6 +9,8 @@ type Ignore = ReturnType<typeof ignoreModule>;
 
 // Files that must never be pushed, deleted, or overwritten on the server via CLI.
 export const PROTECTED_FILES = new Set(['.realm.json']);
+const DELETE_TIMEOUT_MS = 10_000;
+const DELETE_TIMEOUT_PROBE_MS = 3_000;
 
 export function isProtectedFile(relativePath: string): boolean {
   const normalizedPath = relativePath.replace(/\\/g, '/').replace(/^\/+/, '');
@@ -50,6 +52,15 @@ export interface SyncOptions {
   realmUrl: string;
   localDir: string;
   dryRun?: boolean;
+  /**
+   * Append `?waitForIndex=true` to the `_atomic` POST so the realm-server
+   * returns only after the indexer has processed the batch. The
+   * `_atomic` handler hardcoded `waitForIndex: false` after CS-11003
+   * PR 2 (deferred `+source` POST), so callers that read indexed state
+   * (search / list) immediately after a sync race the indexer. Off by
+   * default.
+   */
+  waitForIndex?: boolean;
 }
 
 const REMOTE_CONCURRENCY = 10;
@@ -453,7 +464,9 @@ export abstract class RealmSyncBase {
       }),
     );
 
-    const url = `${this.normalizedRealmUrl}_atomic`;
+    const url = this.options.waitForIndex
+      ? `${this.normalizedRealmUrl}_atomic?waitForIndex=true`
+      : `${this.normalizedRealmUrl}_atomic`;
     const response = await this.authenticator.authedRealmFetch(url, {
       method: 'POST',
       headers: {
@@ -567,13 +580,45 @@ export abstract class RealmSyncBase {
     }
 
     const url = this.buildFileUrl(relativePath);
+    const startedAt = Date.now();
 
-    const response = await this.authenticator.authedRealmFetch(url, {
-      method: 'DELETE',
-      headers: {
-        Accept: SupportedMimeType.CardSource,
-      },
-    });
+    let response: Response;
+    try {
+      response = await this.authenticator.authedRealmFetch(url, {
+        method: 'DELETE',
+        headers: {
+          Accept: SupportedMimeType.CardSource,
+        },
+        signal: AbortSignal.timeout(DELETE_TIMEOUT_MS),
+      });
+    } catch (error) {
+      let elapsedMs = Date.now() - startedAt;
+      console.error(
+        `  Delete request failed after ${elapsedMs}ms: ${relativePath}`,
+      );
+      if (
+        error instanceof Error &&
+        (error.name === 'TimeoutError' || error.name === 'AbortError')
+      ) {
+        let deleteApplied = await this.verifyDeleteApplied(relativePath);
+        if (deleteApplied === true) {
+          console.warn(
+            `  Delete response timed out after ${DELETE_TIMEOUT_MS}ms, but ${relativePath} is already gone on the realm; continuing`,
+          );
+          return;
+        }
+        throw new Error(
+          `Timed out deleting ${relativePath} after ${DELETE_TIMEOUT_MS}ms`,
+          { cause: error },
+        );
+      }
+      throw error;
+    }
+
+    let elapsedMs = Date.now() - startedAt;
+    console.log(
+      `  Delete response for ${relativePath}: ${response.status} ${response.statusText} (${elapsedMs}ms)`,
+    );
 
     if (!response.ok && response.status !== 404) {
       throw new Error(
@@ -584,6 +629,30 @@ export abstract class RealmSyncBase {
     console.log(`  Deleted: ${relativePath}`);
   }
 
+  private async verifyDeleteApplied(
+    relativePath: string,
+  ): Promise<boolean | 'unknown'> {
+    const url = this.buildFileUrl(relativePath);
+    try {
+      const response = await this.authenticator.authedRealmFetch(url, {
+        headers: {
+          Accept: SupportedMimeType.CardSource,
+        },
+        signal: AbortSignal.timeout(DELETE_TIMEOUT_PROBE_MS),
+      });
+      console.warn(
+        `  Delete-timeout probe for ${relativePath}: ${response.status} ${response.statusText}`,
+      );
+      return response.status === 404 ? true : false;
+    } catch (probeError) {
+      console.warn(
+        `  Delete-timeout probe failed for ${relativePath}:`,
+        probeError,
+      );
+      return 'unknown';
+    }
+  }
+
   protected async deleteLocalFile(localPath: string): Promise<void> {
     console.log(`Deleting local: ${localPath}`);