@cardstack/boxel-cli 0.0.1

package/src/commands/realm/pull.ts

@@ -0,0 +1,245 @@
+import type { Command } from 'commander';
+import { RealmSyncBase, type SyncOptions } from '../../lib/realm-sync-base';
+import {
+  CheckpointManager,
+  type CheckpointChange,
+} from '../../lib/checkpoint-manager';
+import {
+  getProfileManager,
+  type ProfileManager,
+} from '../../lib/profile-manager';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+interface PullOptions extends SyncOptions {
+  deleteLocal?: boolean;
+}
+
+class RealmPuller extends RealmSyncBase {
+  hasError = false;
+  downloadedFiles: string[] = [];
+
+  constructor(
+    private pullOptions: PullOptions,
+    profileManager: ProfileManager,
+  ) {
+    super(pullOptions, profileManager);
+  }
+
+  async sync(): Promise<void> {
+    console.log(
+      `Starting pull from ${this.options.realmUrl} to ${this.options.localDir}`,
+    );
+
+    console.log('Testing realm access...');
+    try {
+      await this.getRemoteFileList('');
+    } catch (error) {
+      console.error('Failed to access realm:', error);
+      throw new Error(
+        'Cannot proceed with pull: Authentication or access failed. ' +
+          'Please check your Matrix credentials and realm permissions.',
+      );
+    }
+    console.log('Realm access verified');
+
+    const [remoteFiles, localFiles] = await Promise.all([
+      this.getRemoteFileList(),
+      this.getLocalFileList(),
+    ]);
+    console.log(`Found ${remoteFiles.size} files in remote realm`);
+    console.log(`Found ${localFiles.size} files in local directory`);
+
+    if (this.options.dryRun) {
+      try {
+        await fs.access(this.options.localDir);
+      } catch {
+        console.log(
+          `[DRY RUN] Would create directory: ${this.options.localDir}`,
+        );
+      }
+    } else {
+      await fs.mkdir(this.options.localDir, { recursive: true });
+    }
+
+    const filesToDelete = new Set<string>();
+    if (this.pullOptions.deleteLocal) {
+      for (const relativePath of localFiles.keys()) {
+        if (!remoteFiles.has(relativePath)) {
+          filesToDelete.add(relativePath);
+        }
+      }
+    }
+
+    const checkpointManager = new CheckpointManager(this.options.localDir);
+
+    if (filesToDelete.size > 0 && !this.options.dryRun) {
+      const deleteChanges: CheckpointChange[] = Array.from(filesToDelete).map(
+        (f) => ({
+          file: f,
+          status: 'deleted' as const,
+        }),
+      );
+      const preDeleteCheckpoint = await checkpointManager.createCheckpoint(
+        'remote',
+        deleteChanges,
+        `Pre-delete checkpoint: ${filesToDelete.size} files not on server`,
+      );
+      if (preDeleteCheckpoint) {
+        console.log(
+          `\nCheckpoint created before deletion: ${preDeleteCheckpoint.shortHash}`,
+        );
+      }
+    }
+
+    const downloadResults = await Promise.all(
+      Array.from(remoteFiles.keys()).map((relativePath) =>
+        this.remoteLimit(async () => {
+          try {
+            const localPath = path.join(this.options.localDir, relativePath);
+            await this.downloadFile(relativePath, localPath);
+            return relativePath;
+          } catch (error) {
+            this.hasError = true;
+            console.error(`Error downloading ${relativePath}:`, error);
+            return null;
+          }
+        }),
+      ),
+    );
+    this.downloadedFiles = downloadResults.filter(
+      (f): f is string => f !== null,
+    );
+
+    let deletedFiles: string[] = [];
+    if (filesToDelete.size > 0) {
+      console.log(
+        `\nDeleting ${filesToDelete.size} local files that don't exist in realm...`,
+      );
+
+      const deleteResults = await Promise.all(
+        Array.from(filesToDelete).map(async (relativePath) => {
+          try {
+            const localPath = localFiles.get(relativePath);
+            if (localPath) {
+              await this.deleteLocalFile(localPath);
+              console.log(`  Deleted: ${relativePath}`);
+              return relativePath;
+            }
+            return null;
+          } catch (error) {
+            this.hasError = true;
+            console.error(`Error deleting local file ${relativePath}:`, error);
+            return null;
+          }
+        }),
+      );
+      deletedFiles = deleteResults.filter((f): f is string => f !== null);
+    }
+
+    if (
+      !this.options.dryRun &&
+      this.downloadedFiles.length + deletedFiles.length > 0
+    ) {
+      const pullChanges: CheckpointChange[] = [
+        ...this.downloadedFiles.map((f) => ({
+          file: f,
+          status: 'modified' as const,
+        })),
+        ...deletedFiles.map((f) => ({
+          file: f,
+          status: 'deleted' as const,
+        })),
+      ];
+      const checkpoint = await checkpointManager.createCheckpoint(
+        'remote',
+        pullChanges,
+      );
+      if (checkpoint) {
+        const tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]';
+        console.log(
+          `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`,
+        );
+      }
+    }
+
+    console.log('Pull completed');
+  }
+}
+
+export interface PullCommandOptions {
+  delete?: boolean;
+  dryRun?: boolean;
+  profileManager?: ProfileManager;
+}
+
+export function registerPullCommand(realm: Command): void {
+  realm
+    .command('pull')
+    .description('Pull files from a Boxel realm to a local directory')
+    .argument(
+      '<realm-url>',
+      'The URL of the source realm (e.g., https://app.boxel.ai/demo/)',
+    )
+    .argument('<local-dir>', 'The local directory to sync files to')
+    .option('--delete', 'Delete local files that do not exist in the realm')
+    .option('--dry-run', 'Show what would be done without making changes')
+    .action(
+      async (
+        realmUrl: string,
+        localDir: string,
+        options: { delete?: boolean; dryRun?: boolean },
+      ) => {
+        let result = await pull(realmUrl, localDir, options);
+        if (result.error) {
+          console.error(`Error: ${result.error}`);
+          process.exit(result.files.length > 0 ? 2 : 1);
+        }
+        console.log('Pull completed successfully');
+      },
+    );
+}
+
+export async function pull(
+  realmUrl: string,
+  localDir: string,
+  options: PullCommandOptions,
+): Promise<{ files: string[]; error?: string }> {
+  let pm = options.profileManager ?? getProfileManager();
+  let active = pm.getActiveProfile();
+  if (!active) {
+    return {
+      files: [],
+      error: 'No active profile. Run `boxel profile add` to create one.',
+    };
+  }
+
+  try {
+    const puller = new RealmPuller(
+      {
+        realmUrl,
+        localDir,
+        deleteLocal: options.delete,
+        dryRun: options.dryRun,
+      },
+      pm,
+    );
+
+    await puller.sync();
+
+    if (puller.hasError) {
+      return {
+        files: puller.downloadedFiles.sort(),
+        error:
+          'Pull completed with errors. Some files may not have been downloaded.',
+      };
+    }
+
+    return { files: puller.downloadedFiles.sort() };
+  } catch (error) {
+    return {
+      files: [],
+      error: `Pull failed: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}

package/src/commands/realm/push.ts

@@ -0,0 +1,379 @@
+import type { Command } from 'commander';
+import {
+  RealmSyncBase,
+  isProtectedFile,
+  type SyncOptions,
+} from '../../lib/realm-sync-base';
+import {
+  CheckpointManager,
+  type CheckpointChange,
+} from '../../lib/checkpoint-manager';
+import {
+  getProfileManager,
+  type ProfileManager,
+} from '../../lib/profile-manager';
+import {
+  type SyncManifest,
+  computeFileHash,
+  loadManifest,
+  saveManifest,
+  pathExists,
+} from '../../lib/sync-manifest';
+
+interface PushOptions extends SyncOptions {
+  deleteRemote?: boolean;
+  force?: boolean;
+}
+
+class RealmPusher extends RealmSyncBase {
+  hasError = false;
+
+  constructor(
+    private pushOptions: PushOptions,
+    profileManager: ProfileManager,
+  ) {
+    super(pushOptions, profileManager);
+  }
+
+  async sync(): Promise<void> {
+    console.log(
+      `Starting push from ${this.options.localDir} to ${this.options.realmUrl}`,
+    );
+
+    console.log('Testing realm access...');
+    let initialRemoteFiles: Map<string, boolean>;
+    try {
+      initialRemoteFiles = await this.getRemoteFileList('');
+    } catch (error) {
+      console.error('Failed to access realm:', error);
+      throw new Error(
+        'Cannot proceed with push: Authentication or access failed. ' +
+          'Please check your credentials and realm permissions.',
+      );
+    }
+    console.log('Realm access verified');
+
+    const localFiles = await this.getLocalFileList();
+    console.log(`Found ${localFiles.size} files in local directory`);
+
+    const manifest = await loadManifest(this.options.localDir);
+    const newManifest: SyncManifest = {
+      realmUrl: this.normalizedRealmUrl,
+      files: {},
+      remoteMtimes: {},
+    };
+
+    const filesToUpload: Map<string, string> = new Map();
+    const driftedPaths: Set<string> = new Set();
+
+    const canGoIncremental =
+      !this.pushOptions.force &&
+      manifest !== null &&
+      manifest.realmUrl === this.normalizedRealmUrl;
+
+    if (!canGoIncremental) {
+      if (this.pushOptions.force) {
+        console.log('Force mode: uploading all files');
+      } else if (!manifest) {
+        console.log('No sync manifest found, will upload all files');
+      } else {
+        console.log('Realm URL changed, will upload all files');
+      }
+      for (const [relativePath, localPath] of localFiles) {
+        if (isProtectedFile(relativePath)) continue;
+        filesToUpload.set(relativePath, localPath);
+      }
+    } else {
+      console.log('Checking for changed files...');
+      let skipped = 0;
+
+      const [remoteMtimes, hashResults] = await Promise.all([
+        this.getRemoteMtimes(),
+        Promise.all(
+          Array.from(localFiles.entries()).map(
+            async ([relativePath, localPath]) => {
+              if (isProtectedFile(relativePath)) {
+                return {
+                  relativePath,
+                  localPath,
+                  currentHash: '',
+                  protected: true,
+                };
+              }
+              const currentHash = await computeFileHash(localPath);
+              return {
+                relativePath,
+                localPath,
+                currentHash,
+                protected: false,
+              };
+            },
+          ),
+        ),
+      ]);
+
+      for (const entry of hashResults) {
+        if (entry.protected) {
+          skipped++;
+          continue;
+        }
+        const previousHash = manifest!.files[entry.relativePath];
+        const prevMtime = manifest!.remoteMtimes?.[entry.relativePath];
+        const currMtime = remoteMtimes.get(entry.relativePath);
+
+        const localChanged = previousHash !== entry.currentHash;
+        const remoteMissing =
+          previousHash !== undefined &&
+          !initialRemoteFiles.has(entry.relativePath);
+        const remoteMtimeChanged =
+          prevMtime !== undefined &&
+          currMtime !== undefined &&
+          currMtime !== prevMtime;
+
+        if (localChanged || remoteMissing || remoteMtimeChanged) {
+          filesToUpload.set(entry.relativePath, entry.localPath);
+          if (!localChanged && (remoteMissing || remoteMtimeChanged)) {
+            driftedPaths.add(entry.relativePath);
+          }
+        } else {
+          skipped++;
+          newManifest.files[entry.relativePath] = entry.currentHash;
+          if (prevMtime !== undefined) {
+            newManifest.remoteMtimes![entry.relativePath] = prevMtime;
+          }
+        }
+      }
+
+      if (skipped > 0) {
+        console.log(`Skipping ${skipped} unchanged files`);
+      }
+
+      if (driftedPaths.size > 0) {
+        const list = Array.from(driftedPaths);
+        const preview = list.slice(0, 5).join(', ');
+        const suffix = list.length > 5 ? ', ...' : '';
+        console.warn(
+          `Warning: ${driftedPaths.size} file(s) changed on the realm since your last push; your local versions will overwrite them: ${preview}${suffix}`,
+        );
+      }
+    }
+
+    let uploadFailed = false;
+
+    if (filesToUpload.size === 0) {
+      console.log('No files to upload - everything is up to date');
+    } else {
+      console.log(`Uploading ${filesToUpload.size} file(s) via /_atomic...`);
+
+      // Choose `op: add` vs `op: update` per file. When we have a
+      // manifest, the choice reflects our *intent* so the atomic
+      // endpoint can surface concurrent creation (409) and concurrent
+      // deletion (404):
+      //   - File not in our manifest         →  op: add
+      //   - File in manifest, remote-missing →  op: add (drift re-create)
+      //   - File in manifest, on the remote  →  op: update
+      // With `--force`, or when there is no manifest (first push, or
+      // recovery from a malformed manifest), we defer to the actual
+      // remote state to avoid spurious 409/404s from intent the user
+      // never expressed.
+      const addPaths = new Set<string>();
+      const deferToRemote = this.pushOptions.force || !manifest;
+      for (const relativePath of filesToUpload.keys()) {
+        if (deferToRemote) {
+          if (!initialRemoteFiles.has(relativePath)) {
+            addPaths.add(relativePath);
+          }
+        } else {
+          const knownToManifest = manifest!.files[relativePath] !== undefined;
+          const knownMissing =
+            knownToManifest && !initialRemoteFiles.has(relativePath);
+          if (!knownToManifest || knownMissing) {
+            addPaths.add(relativePath);
+          }
+        }
+      }
+
+      const result = await this.uploadFilesAtomic(filesToUpload, addPaths);
+
+      if (result.error) {
+        uploadFailed = true;
+        this.hasError = true;
+        console.error(result.error.message);
+        for (const entry of result.error.perFile) {
+          let hint: string;
+          if (entry.status === 409) {
+            hint = `${entry.path} was created on the realm concurrently — run with --force to overwrite.`;
+          } else if (entry.status === 404) {
+            hint = `${entry.path} was removed from the realm concurrently — run with --force to re-create it from your local copy.`;
+          } else {
+            hint = `${entry.path}: ${entry.title}`;
+          }
+          console.error(`  ${hint}`);
+        }
+      } else if (result.succeeded.length > 0) {
+        const uploaded = await Promise.all(
+          result.succeeded.map(async (rel) => ({
+            rel,
+            hash: await computeFileHash(filesToUpload.get(rel)!),
+          })),
+        );
+        for (const { rel, hash } of uploaded) {
+          newManifest.files[rel] = hash;
+        }
+      }
+    }
+
+    if (this.pushOptions.deleteRemote) {
+      const filesToDelete = new Set(initialRemoteFiles.keys());
+
+      for (const relativePath of filesToDelete) {
+        if (isProtectedFile(relativePath)) {
+          filesToDelete.delete(relativePath);
+        }
+      }
+
+      for (const relativePath of localFiles.keys()) {
+        filesToDelete.delete(relativePath);
+      }
+
+      if (filesToDelete.size > 0) {
+        console.log(
+          `Deleting ${filesToDelete.size} remote files that don't exist locally`,
+        );
+
+        await Promise.all(
+          Array.from(filesToDelete).map(async (relativePath) => {
+            try {
+              await this.deleteFile(relativePath);
+            } catch (error) {
+              this.hasError = true;
+              console.error(`Error deleting ${relativePath}:`, error);
+            }
+          }),
+        );
+      }
+    }
+
+    if (!this.options.dryRun && !uploadFailed && filesToUpload.size > 0) {
+      try {
+        const freshMtimes = await this.getRemoteMtimes();
+        for (const rel of Object.keys(newManifest.files)) {
+          const mtime = freshMtimes.get(rel);
+          if (mtime !== undefined) {
+            newManifest.remoteMtimes![rel] = mtime;
+          }
+        }
+      } catch (error) {
+        console.warn('Could not refresh remote mtimes after upload:', error);
+      }
+    }
+
+    if (
+      newManifest.remoteMtimes &&
+      Object.keys(newManifest.remoteMtimes).length === 0
+    ) {
+      delete newManifest.remoteMtimes;
+    }
+
+    if (!this.options.dryRun && !uploadFailed) {
+      await saveManifest(this.options.localDir, newManifest);
+    }
+
+    if (!this.options.dryRun && filesToUpload.size > 0 && !uploadFailed) {
+      const checkpointManager = new CheckpointManager(this.options.localDir);
+      const pushChanges: CheckpointChange[] = Array.from(
+        filesToUpload.keys(),
+      ).map((f) => ({
+        file: f,
+        status: 'modified' as const,
+      }));
+      const checkpoint = await checkpointManager.createCheckpoint(
+        'local',
+        pushChanges,
+      );
+      if (checkpoint) {
+        const tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]';
+        console.log(
+          `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`,
+        );
+      }
+    }
+
+    console.log('Push completed');
+  }
+}
+
+export interface PushCommandOptions {
+  delete?: boolean;
+  dryRun?: boolean;
+  force?: boolean;
+  profileManager?: ProfileManager;
+}
+
+export function registerPushCommand(realm: Command): void {
+  realm
+    .command('push')
+    .description('Push local files to a Boxel realm')
+    .argument('<local-dir>', 'The local directory containing files to sync')
+    .argument(
+      '<realm-url>',
+      'The URL of the target realm (e.g., https://app.boxel.ai/demo/)',
+    )
+    .option('--delete', 'Delete remote files that do not exist locally')
+    .option('--dry-run', 'Show what would be done without making changes')
+    .option('--force', 'Upload all files, even if unchanged')
+    .action(
+      async (
+        localDir: string,
+        realmUrl: string,
+        options: { delete?: boolean; dryRun?: boolean; force?: boolean },
+      ) => {
+        await pushCommand(localDir, realmUrl, options);
+      },
+    );
+}
+
+export async function pushCommand(
+  localDir: string,
+  realmUrl: string,
+  options: PushCommandOptions,
+): Promise<void> {
+  let pm = options.profileManager ?? getProfileManager();
+  let active = pm.getActiveProfile();
+  if (!active) {
+    console.error(
+      'Error: no active profile. Run `boxel profile add` to create one.',
+    );
+    process.exit(1);
+  }
+
+  if (!(await pathExists(localDir))) {
+    console.error(`Local directory does not exist: ${localDir}`);
+    process.exit(1);
+  }
+
+  try {
+    const pusher = new RealmPusher(
+      {
+        realmUrl,
+        localDir,
+        deleteRemote: options.delete,
+        dryRun: options.dryRun,
+        force: options.force,
+      },
+      pm,
+    );
+
+    await pusher.sync();
+
+    if (pusher.hasError) {
+      console.log('Push did not complete successfully. View logs for details');
+      process.exit(2);
+    } else {
+      console.log('Push completed successfully');
+    }
+  } catch (error) {
+    console.error('Push failed:', error);
+    process.exit(1);
+  }
+}