@cardstack/boxel-cli 0.2.0-unstable.446 → 0.2.0-unstable.448

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cardstack/boxel-cli",
-  "version": "0.2.0-unstable.446",
+  "version": "0.2.0-unstable.448",
   "license": "MIT",
   "description": "CLI tools for Boxel workspace management",
   "main": "./dist/index.js",
@@ -53,9 +53,9 @@
     "typescript": "~5.9.3",
     "vite": "^6.3.2",
     "vitest": "^2.1.9",
+    "@cardstack/local-types": "0.0.0",
     "@cardstack/postgres": "0.0.0",
-    "@cardstack/runtime-common": "1.0.0",
-    "@cardstack/local-types": "0.0.0"
+    "@cardstack/runtime-common": "1.0.0"
   },
   "publishConfig": {
     "access": "public",

package/src/commands/file/read.ts

@@ -6,14 +6,21 @@ import {
 } from '../../lib/profile-manager';
 import { ensureTrailingSlash } from '@cardstack/runtime-common/paths';
 import { SupportedMimeType } from '@cardstack/runtime-common/supported-mime-type';
+import { isBinaryFilename } from '@cardstack/runtime-common/infer-content-type';
 import { FG_RED, DIM, RESET } from '../../lib/colors';
 import { cliLog } from '../../lib/cli-log';
 
 export interface ReadResult {
   ok: boolean;
   status?: number;
-  /** Raw text content of the file. */
+  /** Raw text content of the file. Populated for non-binary paths. */
   content?: string;
+  /**
+   * Raw bytes. Populated when the requested path is a binary filename
+   * (PNG, PDF, font, etc.) — see `isBinaryFilename`. Mutually exclusive
+   * with `content`.
+   */
+  bytes?: Uint8Array;
   error?: string;
 }
 
@@ -27,8 +34,10 @@ interface ReadCliOptions {
 }
 
 /**
- * Read a file from a realm. Always returns the raw text content.
- * Callers should parse the content themselves if needed (e.g. JSON).
+ * Read a file from a realm. Returns raw text in `content` for text files;
+ * returns raw bytes in `bytes` for binary files (PNG / PDF / font / etc.,
+ * per `isBinaryFilename`). Callers should parse the content themselves
+ * if needed (e.g. JSON).
  *
  * Uses the per-realm JWT via `ProfileManager.authedRealmFetch`.
  */
@@ -70,6 +79,11 @@ export async function read(
     };
   }
 
+  if (isBinaryFilename(path)) {
+    let bytes = new Uint8Array(await response.arrayBuffer());
+    return { ok: true, status: response.status, bytes };
+  }
+
   let text = await response.text();
   return { ok: true, status: response.status, content: text };
 }
@@ -96,9 +110,30 @@ export function registerReadCommand(parent: Command): void {
       }
 
       if (opts.json) {
-        cliLog.output(JSON.stringify(result, null, 2));
+        let serializable: Record<string, unknown> = {
+          ok: result.ok,
+          status: result.status,
+          error: result.error,
+        };
+        if (result.content !== undefined) {
+          serializable.content = result.content;
+        }
+        if (result.bytes !== undefined) {
+          // Buffer.from(typedArray) shares memory, then toString('base64')
+          // copies into a base64 string — fine for the JSON output path.
+          serializable.bytesBase64 = Buffer.from(
+            result.bytes.buffer,
+            result.bytes.byteOffset,
+            result.bytes.byteLength,
+          ).toString('base64');
+        }
+        cliLog.output(JSON.stringify(serializable, null, 2));
       } else if (result.ok) {
-        cliLog.output(result.content ?? '');
+        if (result.bytes !== undefined) {
+          process.stdout.write(result.bytes);
+        } else {
+          cliLog.output(result.content ?? '');
+        }
       } else {
         console.error(
           `${DIM}Status:${RESET} ${result.status ?? '(no status)'}`,

package/src/commands/file/write.ts

@@ -7,6 +7,7 @@ import {
 } from '../../lib/profile-manager';
 import { ensureTrailingSlash } from '@cardstack/runtime-common/paths';
 import { SupportedMimeType } from '@cardstack/runtime-common/supported-mime-type';
+import { isBinaryFilename } from '@cardstack/runtime-common/infer-content-type';
 import { FG_GREEN, FG_RED, DIM, RESET } from '../../lib/colors';
 import { cliLog } from '../../lib/cli-log';
 
@@ -26,15 +27,19 @@ interface WriteCliOptions {
 }
 
 /**
- * Write a file to a realm. Content is sent as-is with card+source MIME type.
- * Path should include the file extension.
+ * Write a file to a realm. Path should include the file extension.
+ *
+ * String content is sent with the card+source MIME type (the text path
+ * .gts / .json / .md / etc. always took). Binary content (a `Uint8Array`,
+ * including the `Buffer` subclass) is sent with `application/octet-stream`,
+ * which the realm-server routes to `upsertBinaryFile` and writes verbatim.
  *
  * Uses the per-realm JWT via `ProfileManager.authedRealmFetch`.
  */
 export async function write(
   realmUrl: string,
   path: string,
-  content: string,
+  content: string | Uint8Array,
   options?: WriteCommandOptions,
 ): Promise<WriteResult> {
   let pm = options?.profileManager ?? getProfileManager();
@@ -47,15 +52,38 @@ export async function write(
   }
 
   let url = new URL(path, ensureTrailingSlash(realmUrl)).href;
+  let isBinary = typeof content !== 'string';
+
+  // Defense-in-depth for programmatic callers (BoxelClient.write, tests).
+  // The CLI wrapper has an earlier guard against `--file image.png` →
+  // `notes.md` style misuse, but the library function is also reachable
+  // without going through that branch. Reject the mismatch here so raw
+  // bytes never land at a text extension (corrupt-on-read) and a UTF-8
+  // string never lands at a binary extension (corrupt-on-write).
+  let pathIsBinary = isBinaryFilename(path);
+  if (pathIsBinary !== isBinary) {
+    return {
+      ok: false,
+      error:
+        `Path ${path} is ${pathIsBinary ? 'binary' : 'text'} by extension ` +
+        `but content is ${isBinary ? 'bytes' : 'a string'}. ` +
+        `Refusing to write to avoid silent corruption.`,
+    };
+  }
 
   try {
     let response = await pm.authedRealmFetch(url, {
       method: 'POST',
-      headers: {
-        Accept: SupportedMimeType.CardSource,
-        'Content-Type': SupportedMimeType.CardSource,
-      },
-      body: content,
+      headers: isBinary
+        ? { 'Content-Type': SupportedMimeType.OctetStream }
+        : {
+            Accept: SupportedMimeType.CardSource,
+            'Content-Type': SupportedMimeType.CardSource,
+          },
+      // Both branches of `content: string | Uint8Array` are valid
+      // BodyInit values, but TS narrows them as a union that doesn't
+      // unify against the fetch signature without a hint.
+      body: content as BodyInit,
     });
 
     if (!response.ok) {
@@ -103,10 +131,30 @@ export function registerWriteCommand(parent: Command): void {
     )
     .option('--json', 'Output raw JSON response')
     .action(async (filePath: string, opts: WriteCliOptions) => {
-      let content: string;
+      let content: string | Uint8Array;
       if (opts.file) {
+        // Refuse a source/destination binary-classification mismatch
+        // (e.g., `write notes.md --file image.png`) — otherwise raw
+        // bytes would land at a text extension and corrupt-on-read.
+        const srcIsBinary = isBinaryFilename(opts.file);
+        const dstIsBinary = isBinaryFilename(filePath);
+        if (srcIsBinary !== dstIsBinary) {
+          stderr(
+            `${FG_RED}Error:${RESET} source file ${opts.file} is ${
+              srcIsBinary ? 'binary' : 'text'
+            } but destination path ${filePath} is ${
+              dstIsBinary ? 'binary' : 'text'
+            }. Refusing to write to avoid silent corruption — rename the destination to match.`,
+          );
+          process.exit(1);
+        }
         try {
-          content = readFileSync(opts.file, 'utf-8');
+          // Binary source files are read as raw bytes so write() can
+          // hand them to the realm unchanged; forcing utf-8 would
+          // corrupt PNG / PDF / font / etc. payloads silently.
+          content = srcIsBinary
+            ? readFileSync(opts.file)
+            : readFileSync(opts.file, 'utf-8');
         } catch (err) {
           stderr(
             `${FG_RED}Error:${RESET} Could not read file: ${err instanceof Error ? err.message : String(err)}`,

package/src/commands/realm/push.ts

@@ -200,6 +200,23 @@ class RealmPusher extends RealmSyncBase {
 
       const result = await this.uploadFilesAtomic(filesToUpload, addPaths);
 
+      // Record every file the server actually wrote before surfacing
+      // errors. uploadFilesAtomic can return both `succeeded` and
+      // `error` when the atomic text batch lands but a per-file
+      // binary POST fails — dropping the manifest update in that
+      // case would force a re-add on the next push (409 cascade).
+      if (result.succeeded.length > 0) {
+        const uploaded = await Promise.all(
+          result.succeeded.map(async (rel) => ({
+            rel,
+            hash: await computeFileHash(filesToUpload.get(rel)!),
+          })),
+        );
+        for (const { rel, hash } of uploaded) {
+          newManifest.files[rel] = hash;
+        }
+      }
+
       if (result.error) {
         uploadFailed = true;
         this.hasError = true;
@@ -215,16 +232,6 @@ class RealmPusher extends RealmSyncBase {
           }
           console.error(`  ${hint}`);
         }
-      } else if (result.succeeded.length > 0) {
-        const uploaded = await Promise.all(
-          result.succeeded.map(async (rel) => ({
-            rel,
-            hash: await computeFileHash(filesToUpload.get(rel)!),
-          })),
-        );
-        for (const { rel, hash } of uploaded) {
-          newManifest.files[rel] = hash;
-        }
       }
     }
 
@@ -270,7 +277,11 @@ class RealmPusher extends RealmSyncBase {
       }
     }
 
-    if (!this.options.dryRun && !uploadFailed && filesToUpload.size > 0) {
+    // Refresh mtimes and save the manifest even on partial failure —
+    // newManifest.files only contains files the server actually wrote
+    // (unchanged carry-overs + succeeded uploads), so persisting it
+    // is always safe and avoids re-uploading text files that landed.
+    if (!this.options.dryRun && filesToUpload.size > 0) {
       try {
         const freshMtimes = await this.getRemoteMtimes();
         for (const rel of Object.keys(newManifest.files)) {
@@ -291,7 +302,7 @@ class RealmPusher extends RealmSyncBase {
       delete newManifest.remoteMtimes;
     }
 
-    if (!this.options.dryRun && !uploadFailed) {
+    if (!this.options.dryRun) {
       await saveManifest(this.options.localDir, newManifest);
     }
 

package/src/commands/realm/sync.ts

@@ -314,14 +314,16 @@ class RealmSyncer extends RealmSyncBase {
       }
 
       const result = await this.uploadFilesAtomic(filesToUpload, addPaths);
+      // Record every file the server actually wrote, even when other
+      // files in the same batch failed — see push.ts for the symmetric
+      // reasoning.
+      this.pushedFiles.push(...result.succeeded);
       if (result.error) {
         this.hasError = true;
         console.error(result.error.message);
         for (const entry of result.error.perFile) {
           console.error(`  ${entry.path}: ${entry.title}`);
         }
-      } else {
-        this.pushedFiles.push(...result.succeeded);
       }
     }
 
@@ -371,8 +373,12 @@ class RealmSyncer extends RealmSyncBase {
       );
     }
 
-    // Phase 6: Update manifest
-    if (!this.options.dryRun && !this.hasError) {
+    // Phase 6: Update manifest. Persist even on partial failure — we
+    // only record hashes for files the server actually wrote
+    // (pushedFiles + pulledFiles), so the manifest stays consistent
+    // with the realm and the next sync won't re-attempt successful
+    // files.
+    if (!this.options.dryRun) {
       // Build updated hashes from prior manifest + current local files + executed ops.
       // Start with the previous manifest so that files deleted locally but not
       // propagated (no --delete) retain their entries and aren't re-pulled next sync.

package/src/lib/realm-sync-base.ts

@@ -3,6 +3,7 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import ignoreModule from 'ignore';
 import pLimit from 'p-limit';
+import { isBinaryFilename } from '@cardstack/runtime-common/infer-content-type';
 
 const ignore = (ignoreModule as any).default || ignoreModule;
 type Ignore = ReturnType<typeof ignoreModule>;
@@ -42,10 +43,38 @@ function decodeAtomicResultId(id: string): string {
   }
 }
 
+// Builds a structured upload error: the message embeds the response
+// status + statusText + a snippet of the response body (the realm
+// returns useful detail there — size limits, missing scopes, etc.),
+// and a `status` property is attached so the batch helper can route
+// the failure without re-parsing the message.
+async function throwUploadError(
+  response: Response,
+  relativePath: string,
+): Promise<never> {
+  const bodyText = await response.text().catch(() => '');
+  const message = `Failed to upload ${relativePath}: ${response.status} ${response.statusText}${
+    bodyText ? ` — ${bodyText.slice(0, 200)}` : ''
+  }`;
+  const err = new Error(message) as Error & {
+    status?: number;
+    body?: string;
+  };
+  err.status = response.status;
+  err.body = bodyText;
+  throw err;
+}
+
+// Shared shape for per-file upload errors that need to bubble back to
+// callers (push.ts / sync.ts) so they can format hints and decide which
+// successes to persist alongside the failures.
+type UploadFailure = { path: string; status: number; title: string };
+
 export const SupportedMimeType = {
   CardSource: 'application/vnd.card+source',
   DirectoryListing: 'application/vnd.api+json',
   Mtimes: 'application/vnd.api+json',
+  OctetStream: 'application/octet-stream',
 } as const;
 
 export interface SyncOptions {
@@ -396,6 +425,12 @@ export abstract class RealmSyncBase {
       return;
     }
 
+    if (isBinaryFilename(relativePath)) {
+      await this.uploadBinaryFile(relativePath, localPath);
+      console.log(`  Uploaded: ${relativePath}`);
+      return;
+    }
+
     const content = await fs.readFile(localPath, 'utf8');
     const url = this.buildFileUrl(relativePath);
 
@@ -409,14 +444,39 @@ export abstract class RealmSyncBase {
     });
 
     if (!response.ok) {
-      throw new Error(
-        `Failed to upload: ${response.status} ${response.statusText}`,
-      );
+      await throwUploadError(response, relativePath);
     }
 
     console.log(`  Uploaded: ${relativePath}`);
   }
 
+  // Uploads a single binary file (PNG, PDF, font, etc.) per the host
+  // pattern: a per-file POST with Content-Type: application/octet-stream
+  // and the raw bytes as the body. The realm-server routes octet-stream
+  // POSTs to upsertBinaryFile, which writes the bytes verbatim without
+  // any string conversion. Used by both uploadFile (single-shot) and
+  // uploadFilesAtomic (mixed-batch fallback for the binary entries it
+  // splits out of the atomic JSON payload).
+  protected async uploadBinaryFile(
+    relativePath: string,
+    localPath: string,
+  ): Promise<void> {
+    const bytes = await fs.readFile(localPath);
+    const url = this.buildFileUrl(relativePath);
+
+    const response = await this.authenticator.authedRealmFetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': SupportedMimeType.OctetStream,
+      },
+      body: bytes,
+    });
+
+    if (!response.ok) {
+      await throwUploadError(response, relativePath);
+    }
+  }
+
   // Batched upload via the realm's /_atomic endpoint. Returns the set of
   // paths the server reported as written plus an optional error payload
   // when the whole batch was rejected. The atomic endpoint validates
@@ -430,7 +490,7 @@ export abstract class RealmSyncBase {
     succeeded: string[];
     error?: {
       status: number;
-      perFile: Array<{ path: string; status: number; title: string }>;
+      perFile: UploadFailure[];
       message: string;
     };
   }> {
@@ -449,8 +509,137 @@ export abstract class RealmSyncBase {
       return { succeeded: [] };
     }
 
+    // The /_atomic endpoint embeds each file's content inside a JSON
+    // `attributes.content` string, which can't carry raw binary bytes.
+    // Match the host pattern: keep /_atomic for text files only, and
+    // for each binary file fall back to a per-file octet-stream POST
+    // (the same wire format `uploadBinaryFile` uses for a single binary).
+    // The two batches run concurrently — neither helper rejects, so the
+    // outer Promise.all just joins their structured results.
+    const textEntries: Array<[string, string]> = [];
+    const binaryEntries: Array<[string, string]> = [];
+    for (const entry of entries) {
+      if (isBinaryFilename(entry[0])) {
+        binaryEntries.push(entry);
+      } else {
+        textEntries.push(entry);
+      }
+    }
+
+    const [binaryOutcome, textOutcome] = await Promise.all([
+      this.uploadBinaryBatch(binaryEntries),
+      this.uploadTextAtomic(textEntries, addPaths),
+    ]);
+
+    const succeeded = [...textOutcome.succeeded, ...binaryOutcome.succeeded];
+
+    if (textOutcome.fatal) {
+      return {
+        succeeded,
+        error: {
+          status: textOutcome.fatal.status,
+          perFile: [...textOutcome.failed, ...binaryOutcome.failed],
+          message: textOutcome.fatal.message,
+        },
+      };
+    }
+
+    if (binaryOutcome.failed.length > 0) {
+      return {
+        succeeded,
+        error: {
+          status: binaryOutcome.failed[0].status,
+          perFile: binaryOutcome.failed,
+          message: `Binary upload failed for ${binaryOutcome.failed.length} file(s)`,
+        },
+      };
+    }
+
+    return { succeeded };
+  }
+
+  // Fan out the per-file octet-stream POSTs for the binary slice of an
+  // atomic batch. Each upload is wrapped in try/catch so a single failure
+  // is folded into the result instead of rejecting the fan-out;
+  // Promise.allSettled is used at the boundary as defense-in-depth so a
+  // future change that drops the inner catch still surfaces a structured
+  // failure rather than silently aborting other in-flight uploads.
+  private async uploadBinaryBatch(
+    binaryEntries: Array<[string, string]>,
+  ): Promise<{ succeeded: string[]; failed: UploadFailure[] }> {
+    if (binaryEntries.length === 0) {
+      return { succeeded: [], failed: [] };
+    }
+
+    const settled = await Promise.allSettled(
+      binaryEntries.map(([relativePath, localPath]) =>
+        this.remoteLimit(async () => {
+          try {
+            await this.uploadBinaryFile(relativePath, localPath);
+            console.log(`  Uploaded: ${relativePath}`);
+            return { relativePath, ok: true as const };
+          } catch (err) {
+            const errWithStatus = err as { status?: number };
+            const status =
+              typeof errWithStatus?.status === 'number'
+                ? errWithStatus.status
+                : 500;
+            const title = err instanceof Error ? err.message : String(err);
+            return {
+              relativePath,
+              ok: false as const,
+              status,
+              title,
+            };
+          }
+        }),
+      ),
+    );
+
+    const succeeded: string[] = [];
+    const failed: UploadFailure[] = [];
+    for (let i = 0; i < settled.length; i++) {
+      const outcome = settled[i];
+      if (outcome.status === 'fulfilled') {
+        if (outcome.value.ok) {
+          succeeded.push(outcome.value.relativePath);
+        } else {
+          failed.push({
+            path: outcome.value.relativePath,
+            status: outcome.value.status,
+            title: outcome.value.title,
+          });
+        }
+      } else {
+        failed.push({
+          path: binaryEntries[i][0],
+          status: 500,
+          title: String(outcome.reason),
+        });
+      }
+    }
+
+    return { succeeded, failed };
+  }
+
+  // POST the text slice of a mixed batch to /_atomic and decode the
+  // result. `fatal` is set when the server rejected the whole batch
+  // (non-201) — callers map that to a top-level error message; `failed`
+  // carries the per-operation errors the server returned alongside.
+  private async uploadTextAtomic(
+    textEntries: Array<[string, string]>,
+    addPaths: Set<string>,
+  ): Promise<{
+    succeeded: string[];
+    failed: UploadFailure[];
+    fatal?: { status: number; message: string };
+  }> {
+    if (textEntries.length === 0) {
+      return { succeeded: [], failed: [] };
+    }
+
     const operations = await Promise.all(
-      entries.map(async ([relativePath, localPath]) => {
+      textEntries.map(async ([relativePath, localPath]) => {
         const content = await fs.readFile(localPath, 'utf8');
         return {
           op: addPaths.has(relativePath)
@@ -478,27 +667,28 @@ export abstract class RealmSyncBase {
       body: JSON.stringify({ 'atomic:operations': operations }),
     });
 
+    const hrefToRelative = new Map(
+      textEntries.map(([rel]) => [this.buildFileUrl(rel), rel]),
+    );
+
     if (response.status === 201) {
       const body = (await response.json()) as {
         'atomic:results'?: Array<{ data?: { id?: string } }>;
       };
-      const hrefToRelative = new Map(
-        entries.map(([rel]) => [this.buildFileUrl(rel), rel]),
-      );
       // The realm normalizes hrefs: a path with a space goes out as
       // `Knowledge Articles/...` but comes back URL-encoded as
       // `Knowledge%20Articles/...`. Decode the response id before the
       // map lookup so we resolve back to the original relative path
       // instead of falling through to the raw encoded URL.
-      const succeeded = (body['atomic:results'] ?? [])
+      const atomicSucceeded = (body['atomic:results'] ?? [])
         .map((r) => r.data?.id)
         .filter((id): id is string => typeof id === 'string')
         .map((id) => decodeAtomicResultId(id))
         .map((id) => hrefToRelative.get(id) ?? id);
-      for (const rel of succeeded) {
+      for (const rel of atomicSucceeded) {
         console.log(`  Uploaded: ${rel}`);
       }
-      return { succeeded };
+      return { succeeded: atomicSucceeded, failed: [] };
     }
 
     let errorBody: {
@@ -510,15 +700,12 @@ export abstract class RealmSyncBase {
       // ignore JSON parse failures — fall through to the generic message
     }
 
-    const perFile = (errorBody.errors ?? []).map((e) => {
+    const failed: UploadFailure[] = (errorBody.errors ?? []).map((e) => {
       const detail = e.detail ?? '';
       const match = detail.match(/Resource (\S+) /);
       const href = match ? decodeAtomicResultId(match[1]) : '';
-      const relMap = new Map(
-        entries.map(([rel]) => [this.buildFileUrl(rel), rel]),
-      );
       return {
-        path: relMap.get(href) ?? href,
+        path: hrefToRelative.get(href) ?? href,
         status: e.status ?? response.status,
         title: e.title ?? 'Error',
       };
@@ -526,9 +713,9 @@ export abstract class RealmSyncBase {
 
     return {
       succeeded: [],
-      error: {
+      failed,
+      fatal: {
         status: response.status,
-        perFile,
         message: `Atomic upload failed: ${response.status} ${response.statusText}`,
       },
     };
@@ -559,12 +746,16 @@ export abstract class RealmSyncBase {
       );
     }
 
-    const content = await response.text();
-
     const localDir = path.dirname(localPath);
     await fs.mkdir(localDir, { recursive: true });
 
-    await fs.writeFile(localPath, content, 'utf8');
+    if (isBinaryFilename(relativePath)) {
+      const buffer = Buffer.from(await response.arrayBuffer());
+      await fs.writeFile(localPath, buffer);
+    } else {
+      const content = await response.text();
+      await fs.writeFile(localPath, content, 'utf8');
+    }
     console.log(`  Downloaded: ${relativePath}`);
   }