@cardstack/boxel-cli 0.3.2-unstable.61 → 0.4.0-unstable.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cardstack/boxel-cli",
-  "version": "0.3.2-unstable.61",
+  "version": "0.4.0-unstable.1",
   "license": "MIT",
   "description": "CLI tools for Boxel workspace management",
   "main": "./dist/index.js",
@@ -62,9 +62,9 @@
     "ts-node": "^10.9.1",
     "vite": "^6.3.2",
     "vitest": "^2.1.9",
-    "@cardstack/local-types": "0.0.0",
     "@cardstack/postgres": "0.0.0",
-    "@cardstack/runtime-common": "1.0.0"
+    "@cardstack/runtime-common": "1.0.0",
+    "@cardstack/local-types": "0.0.0"
   },
   "publishConfig": {
     "access": "public",

package/src/commands/realm/index.ts

@@ -3,6 +3,7 @@ import { registerCancelIndexingCommand } from './cancel-indexing.ts';
 import { registerCreateCommand } from './create.ts';
 import { registerHistoryCommand } from './history.ts';
 import { registerIndexingErrorsCommand } from './indexing-errors.ts';
+import { registerIngestCardCommand } from './ingest-card.ts';
 import { registerListCommand } from './list.ts';
 import { registerMilestoneCommand } from './milestone.ts';
 import { registerPublishCommand } from './publish.ts';
@@ -24,6 +25,7 @@ export function registerRealmCommand(program: Command): void {
   registerCreateCommand(realm);
   registerHistoryCommand(realm);
   registerIndexingErrorsCommand(realm);
+  registerIngestCardCommand(realm);
   registerListCommand(realm);
   registerMilestoneCommand(realm);
   registerPublishCommand(realm);

package/src/commands/realm/ingest-card.ts

@@ -0,0 +1,567 @@
+import type { Command } from 'commander';
+import * as path from 'path';
+import { ensureTrailingSlash } from '@cardstack/runtime-common/paths';
+import {
+  RealmSyncBase,
+  SupportedMimeType,
+  type SyncOptions,
+} from '../../lib/realm-sync-base.ts';
+import {
+  CheckpointManager,
+  type CheckpointChange,
+} from '../../lib/checkpoint-manager.ts';
+import { resolveRealmAuthenticator } from '../../lib/auth-resolver.ts';
+import { resolveRealmSecretSeed } from '../../lib/prompt.ts';
+import {
+  getProfileManager,
+  type ProfileManager,
+} from '../../lib/profile-manager.ts';
+import type { RealmAuthenticator } from '../../lib/realm-authenticator.ts';
+import { search } from '../search.ts';
+
+const CARD_JSON = 'application/vnd.card+json';
+const MODULE_EXTENSIONS = ['.gts', '.gjs', '.ts', '.js'];
+const SPEC_MODULE = 'https://cardstack.com/base/spec';
+
+/**
+ * The realm index identifies modules without their executable extension;
+ * strip it whenever a module path is used in a type filter or compared
+ * against an indexed module reference.
+ */
+function stripModuleExt(moduleRelOrUrl: string): string {
+  return moduleRelOrUrl.replace(/\.(gts|gjs|ts|js)$/, '');
+}
+
+/**
+ * Pull every module-specifier out of `.gts`/`.ts` source text. Matches the
+ * `from '<spec>'` clause of every `import`/`export … from` (value **and**
+ * `import type`, namespace, re-export) plus side-effect `import '<spec>'`.
+ * Type-only imports matter: they are erased at transpile, so the realm's
+ * runtime dep graph misses them — yet the seeded copy still needs the file
+ * on disk for `boxel parse` to type-check.
+ */
+export function extractImportSpecifiers(source: string): string[] {
+  let specs = new Set<string>();
+  let fromRe = /\bfrom\s*['"]([^'"]+)['"]/g;
+  let sideEffectRe = /\bimport\s*['"]([^'"]+)['"]/g;
+  let m: RegExpExecArray | null;
+  while ((m = fromRe.exec(source))) specs.add(m[1]);
+  while ((m = sideEffectRe.exec(source))) specs.add(m[1]);
+  return [...specs];
+}
+
+/** Exported card/field class names declared in a module (for instance lookup). */
+export function extractExportedClassNames(source: string): string[] {
+  let names = new Set<string>();
+  let re =
+    /\bexport\s+(?:default\s+)?(?:abstract\s+)?class\s+(\w+)\s+extends\b/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(source))) names.add(m[1]);
+  return [...names];
+}
+
+/**
+ * Resolve a same-realm import specifier to a realm-relative file path that
+ * exists in `fileSet`. Handles relative (`./`, `../`), same-origin absolute
+ * (`https://…/<realm>/…`), and registered-prefix (`@cardstack/<realm>/…`)
+ * forms. Returns null for anything outside this realm (base realm, other
+ * realms, bare npm packages) — those resolve at runtime and aren't copied.
+ */
+export function resolveSameRealmFile(
+  spec: string,
+  fromAbsUrl: string,
+  realmRoot: string,
+  fileSet: Set<string>,
+): string | null {
+  let rel: string;
+  if (spec.startsWith('.') || spec.startsWith('/')) {
+    let absUrl = new URL(spec, fromAbsUrl).href;
+    if (!absUrl.startsWith(realmRoot)) return null; // base realm / other realm
+    rel = absUrl.slice(realmRoot.length).replace(/^\/+/, '');
+  } else if (/^https?:\/\//.test(spec)) {
+    if (!spec.startsWith(realmRoot)) return null;
+    rel = spec.slice(realmRoot.length).replace(/^\/+/, '');
+  } else if (spec.startsWith('@')) {
+    // Registered-prefix form (`@cardstack/<realm>/…`). Map onto this realm by
+    // its path tail (e.g. `catalog/`). npm-scoped packages (`@glimmer/…`)
+    // won't contain the realm tail and fall through to null (external).
+    let tail = new URL(realmRoot).pathname.replace(/^\/+/, '');
+    let idx = tail ? spec.indexOf(tail) : -1;
+    if (idx < 0) return null;
+    rel = spec.slice(idx + tail.length).replace(/^\/+/, '');
+  } else {
+    return null; // bare npm module — external
+  }
+  let candidates = MODULE_EXTENSIONS.some((ext) => rel.endsWith(ext))
+    ? [rel]
+    : [rel, ...MODULE_EXTENSIONS.map((ext) => rel + ext)];
+  for (let cand of candidates) {
+    if (fileSet.has(cand)) return cand;
+  }
+  return null;
+}
+
+class RealmCardIngester extends RealmSyncBase {
+  hasError = false;
+  copiedFiles: string[] = [];
+  private profileManager: ProfileManager;
+  private cardUrl: string;
+  private sourceCache = new Map<string, string | null>();
+
+  constructor(
+    options: SyncOptions & { cardUrl: string },
+    authenticator: RealmAuthenticator,
+    profileManager: ProfileManager,
+  ) {
+    super(options, authenticator);
+    this.profileManager = profileManager;
+    this.cardUrl = options.cardUrl;
+  }
+
+  private get realmRoot(): string {
+    return this.normalizedRealmUrl; // always ends with '/'
+  }
+
+  private relToAbs(rel: string): string {
+    return this.realmRoot + rel.replace(/^\/+/, '');
+  }
+
+  private toRel(absUrl: string): string {
+    return absUrl.startsWith(this.realmRoot)
+      ? absUrl.slice(this.realmRoot.length).replace(/^\/+/, '')
+      : absUrl;
+  }
+
+  /**
+   * Normalize any reference to a realm-relative path. The realm is reachable
+   * under two URL spaces: the served root (`https://…/catalog/`, used by file
+   * I/O and `_mtimes`) and a published alias (`@cardstack/catalog/`, used by
+   * search-result `id`s and `ref.module`). Both share the realm's path tail
+   * (`catalog/`), so we relativize against that.
+   */
+  private relativize(url: string): string {
+    if (url.startsWith(this.realmRoot)) {
+      return url.slice(this.realmRoot.length).replace(/^\/+/, '');
+    }
+    let tail = new URL(this.realmRoot).pathname.replace(/^\/+/, ''); // e.g. catalog/
+    let idx = tail ? url.indexOf(tail) : -1;
+    if (idx >= 0) return url.slice(idx + tail.length).replace(/^\/+/, '');
+    return url.replace(/^\/+/, '');
+  }
+
+  /** Resolve a module ref (absolute alias/https or relative) to a realm-relative path. */
+  private refToRel(refModule: string, baseRel: string): string {
+    if (/^https?:\/\//.test(refModule) || refModule.startsWith('@')) {
+      return this.relativize(refModule);
+    }
+    return new URL(refModule, `https://_/${baseRel}`).pathname.replace(
+      /^\/+/,
+      '',
+    );
+  }
+
+  /** Fetch a realm file's source text (cached). Null on non-OK. */
+  private async fetchText(rel: string): Promise<string | null> {
+    if (this.sourceCache.has(rel)) return this.sourceCache.get(rel)!;
+    let res = await this.authenticator.authedRealmFetch(
+      this.buildFileUrl(rel),
+      {
+        headers: { Accept: SupportedMimeType.CardSource },
+      },
+    );
+    let text = res.ok ? await res.text() : null;
+    this.sourceCache.set(rel, text);
+    return text;
+  }
+
+  async sync(): Promise<void> {
+    let cardUrl = this.cardUrl;
+    let fileSet = new Set((await this.getRemoteMtimes()).keys());
+    if (fileSet.size === 0) {
+      throw new Error(
+        `Source realm ${this.realmRoot} returned no files (check the URL and your access).`,
+      );
+    }
+
+    let toCopy = new Set<string>();
+    let entry = await this.resolveEntry(cardUrl, fileSet);
+
+    // 1. Module dependency graph (static imports, transitive, same-realm).
+    let moduleFiles = await this.crawlModules(entry.moduleRels, fileSet);
+    for (let f of moduleFiles) toCopy.add(f);
+
+    // 2. Co-located tests for every seeded module.
+    for (let f of moduleFiles) {
+      let test = f.replace(/\.(gts|gjs|ts|js)$/, '.test.$1');
+      if (test !== f && fileSet.has(test)) toCopy.add(test);
+    }
+
+    // 3. The entry card's own instances.
+    let entryModuleAbs = entry.moduleRels.map((r) => this.relToAbs(r));
+    let instanceRels = await this.findEntryInstances(
+      moduleFiles,
+      entryModuleAbs,
+      fileSet,
+    );
+    for (let r of entry.instanceRels) instanceRels.add(r);
+    for (let r of instanceRels) toCopy.add(r);
+
+    // 4. The card's own Catalog Spec(s) — card/app specType only.
+    let specRels = await this.findCardSpecs(moduleFiles, fileSet);
+    for (let r of specRels) toCopy.add(r);
+
+    if (toCopy.size === 0) {
+      throw new Error(`Nothing to ingest for ${cardUrl}.`);
+    }
+
+    await this.downloadAll([...toCopy], fileSet);
+    await this.writeCheckpoint();
+  }
+
+  /** Classify the entry URL as a module or an instance and seed the crawl. */
+  private async resolveEntry(
+    cardUrl: string,
+    fileSet: Set<string>,
+  ): Promise<{ moduleRels: string[]; instanceRels: string[] }> {
+    let rel = this.toRel(cardUrl.replace(/\/$/, ''));
+    // Instance? Canonical instance URLs omit the `.json` (e.g. `.../Person/1`),
+    // so probe the `.json` file rather than the bare path (which 404s).
+    let instanceRel = rel.endsWith('.json') ? rel : `${rel}.json`;
+    if (fileSet.has(instanceRel)) {
+      let source = await this.fetchText(instanceRel);
+      let adoptsFrom =
+        source != null
+          ? tryParseCardDoc(source)?.data?.meta?.adoptsFrom?.module
+          : undefined;
+      if (adoptsFrom) {
+        // Instance: seed from its definition module, copy the instance.
+        let moduleFile = resolveSameRealmFile(
+          adoptsFrom,
+          this.relToAbs(instanceRel),
+          this.realmRoot,
+          fileSet,
+        );
+        return {
+          moduleRels: moduleFile ? [moduleFile] : [],
+          instanceRels: [instanceRel],
+        };
+      }
+    }
+    // Module: resolve to a real source file.
+    let moduleFile = resolveSameRealmFile(
+      cardUrl,
+      this.realmRoot,
+      this.realmRoot,
+      fileSet,
+    );
+    if (!moduleFile) {
+      throw new Error(
+        `Could not resolve ${cardUrl} to a module or instance in ${this.realmRoot}.`,
+      );
+    }
+    return { moduleRels: [moduleFile], instanceRels: [] };
+  }
+
+  /** BFS the same-realm import graph from the seed modules. */
+  private async crawlModules(
+    seeds: string[],
+    fileSet: Set<string>,
+  ): Promise<Set<string>> {
+    let seen = new Set<string>();
+    let queue = [...seeds];
+    while (queue.length) {
+      let rel = queue.shift()!;
+      if (seen.has(rel)) continue;
+      seen.add(rel);
+      let source = await this.fetchText(rel);
+      if (source == null) continue;
+      for (let spec of extractImportSpecifiers(source)) {
+        let dep = resolveSameRealmFile(
+          spec,
+          this.relToAbs(rel),
+          this.realmRoot,
+          fileSet,
+        );
+        if (dep && !seen.has(dep)) queue.push(dep);
+      }
+    }
+    return seen;
+  }
+
+  /** Instances whose definition is one of the entry module's exported cards. */
+  private async findEntryInstances(
+    moduleFiles: Set<string>,
+    entryModuleAbs: string[],
+    fileSet: Set<string>,
+  ): Promise<Set<string>> {
+    let out = new Set<string>();
+    for (let rel of moduleFiles) {
+      let moduleAbs = this.relToAbs(rel);
+      if (!entryModuleAbs.includes(moduleAbs)) continue;
+      let source = await this.fetchText(rel);
+      if (source == null) continue;
+      for (let name of extractExportedClassNames(source)) {
+        let results = await this.searchCards({
+          filter: { type: { module: stripModuleExt(moduleAbs), name } },
+        });
+        for (let card of results) {
+          let r = this.cardIdToInstanceRel(card.id);
+          if (r && fileSet.has(r)) out.add(r);
+        }
+      }
+    }
+    return out;
+  }
+
+  /** Card/app Spec cards whose `ref` resolves to a seeded module. */
+  private async findCardSpecs(
+    moduleFiles: Set<string>,
+    fileSet: Set<string>,
+  ): Promise<Set<string>> {
+    let out = new Set<string>();
+    let moduleRelsNoExt = new Set([...moduleFiles].map(stripModuleExt));
+    let specs = await this.searchCards({
+      filter: { type: { module: SPEC_MODULE, name: 'Spec' } },
+    });
+    for (let spec of specs) {
+      let attrs = spec.attributes ?? {};
+      let specType = attrs.specType;
+      if (specType !== 'card' && specType !== 'app') continue;
+      let ref = attrs.ref as { module?: string } | undefined;
+      if (!ref?.module || !spec.id) continue;
+      let refRel = stripModuleExt(
+        this.refToRel(ref.module, this.relativize(spec.id)),
+      );
+      if (!moduleRelsNoExt.has(refRel)) continue;
+      let r = this.cardIdToInstanceRel(spec.id);
+      if (r && fileSet.has(r)) out.add(r);
+    }
+    return out;
+  }
+
+  private async searchCards(
+    query: Record<string, unknown>,
+  ): Promise<CardResource[]> {
+    let result = await search([this.realmRoot], query, {
+      profileManager: this.profileManager,
+    });
+    if (!result.ok) {
+      this.hasError = true;
+      console.warn(`  search failed: ${result.error ?? 'unknown'}`);
+      return [];
+    }
+    return (result.data ?? []) as CardResource[];
+  }
+
+  private cardIdToInstanceRel(id: string | undefined): string | null {
+    if (!id) return null;
+    let rel = this.relativize(id);
+    if (!rel) return null;
+    return rel.endsWith('.json') ? rel : `${rel}.json`;
+  }
+
+  private async downloadAll(
+    rels: string[],
+    _fileSet: Set<string>,
+  ): Promise<void> {
+    let results = await Promise.all(
+      rels.map((rel) =>
+        this.remoteLimit(async () => {
+          try {
+            await this.downloadFile(rel, path.join(this.options.localDir, rel));
+            return rel;
+          } catch (error) {
+            this.hasError = true;
+            console.error(`Error downloading ${rel}:`, error);
+            return null;
+          }
+        }),
+      ),
+    );
+    this.copiedFiles = results.filter((r): r is string => r !== null);
+  }
+
+  /**
+   * Best-effort: a checkpoint failure (e.g. unwritable history dir) is
+   * warned about but doesn't fail the ingest — the files are already copied.
+   */
+  private async writeCheckpoint(): Promise<void> {
+    if (this.options.dryRun || this.copiedFiles.length === 0) return;
+    let changes: CheckpointChange[] = this.copiedFiles.map((file) => ({
+      file,
+      status: 'modified' as const,
+    }));
+    try {
+      let checkpoint = await new CheckpointManager(
+        this.options.localDir,
+      ).createCheckpoint(
+        'remote',
+        changes,
+        `Ingest card: ${this.copiedFiles.length} files`,
+      );
+      if (checkpoint) {
+        let tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]';
+        console.log(
+          `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`,
+        );
+      }
+    } catch (error) {
+      console.warn(
+        `Warning: failed to create checkpoint: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+interface CardResource {
+  id?: string;
+  attributes?: { specType?: string; ref?: unknown; [k: string]: unknown };
+}
+
+function tryParseCardDoc(
+  source: string,
+): { data?: { meta?: { adoptsFrom?: { module?: string } } } } | null {
+  try {
+    let parsed = JSON.parse(source);
+    return parsed && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+export interface IngestCardCommandOptions {
+  realm?: string;
+  dryRun?: boolean;
+  realmSecretSeed?: string;
+  profileManager?: ProfileManager;
+  /**
+   * @internal Test hook: supply an already-constructed authenticator,
+   * bypassing both seed resolution and the profile flow.
+   */
+  authenticator?: RealmAuthenticator;
+}
+
+export async function ingestCard(
+  cardUrl: string,
+  localDir: string,
+  options: IngestCardCommandOptions,
+): Promise<{ files: string[]; error?: string }> {
+  // A card URL never ends in a slash; strip one a copy-paste may have added.
+  // It would otherwise skew seed-auth registration (a token scoped to the
+  // card path rather than the realm) and realm auto-detection below.
+  cardUrl = cardUrl.replace(/\/$/, '');
+  let pm = options.profileManager ?? getProfileManager();
+  let resolution = resolveRealmAuthenticator({
+    realmUrl: options.realm ?? cardUrl,
+    realmSecretSeed: options.realmSecretSeed,
+    profileManager: pm,
+    authenticator: options.authenticator,
+  });
+  if (!resolution.ok) return { files: [], error: resolution.error };
+  let authenticator = resolution.authenticator;
+
+  let realmRoot = options.realm
+    ? ensureTrailingSlash(options.realm)
+    : undefined;
+  if (!realmRoot) {
+    try {
+      let res = await authenticator.authedRealmFetch(cardUrl, {
+        headers: { Accept: CARD_JSON },
+      });
+      let header = res.headers.get('x-boxel-realm-url');
+      if (header) realmRoot = ensureTrailingSlash(header);
+    } catch {
+      // fall through to the error below
+    }
+    if (!realmRoot) {
+      return {
+        files: [],
+        error: `Could not determine the source realm for ${cardUrl}. Pass --realm <source-realm-url>.`,
+      };
+    }
+  }
+
+  try {
+    let ingester = new RealmCardIngester(
+      {
+        realmUrl: realmRoot,
+        localDir,
+        dryRun: options.dryRun,
+        cardUrl,
+      },
+      authenticator,
+      pm,
+    );
+    console.log(
+      `Ingesting ${cardUrl}\n  from realm ${realmRoot}\n  into ${localDir}`,
+    );
+    await ingester.sync();
+    console.log(`\nIngested ${ingester.copiedFiles.length} file(s).`);
+    return {
+      files: ingester.copiedFiles.sort(),
+      error: ingester.hasError
+        ? 'Ingest completed with errors — some files may be missing.'
+        : undefined,
+    };
+  } catch (error) {
+    return {
+      files: [],
+      error: `Ingest failed: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+export function registerIngestCardCommand(realm: Command): void {
+  realm
+    .command('ingest-card')
+    .description(
+      'Copy a card from a source realm into a local directory, bundling the ' +
+        "deps that live in the card's own realm: the module, the modules it " +
+        'imports from the same realm, its sample instances, and its Catalog ' +
+        'Spec. Imports that resolve to the base realm or other realms are ' +
+        'left as references (they resolve at runtime). Works across realms; ' +
+        'source and target must be on the same realm server.',
+    )
+    .argument(
+      '<source-card-url>',
+      'Absolute URL of the source card module (or a card instance) to ingest',
+    )
+    .argument(
+      '<local-dir>',
+      'The local directory to write the card + deps into',
+    )
+    .option(
+      '--realm <url>',
+      'Source realm URL (defaults to the realm the card belongs to)',
+    )
+    .option('--dry-run', 'Show what would be copied without writing files')
+    .option(
+      '--realm-secret-seed',
+      'Administrative auth: prompt for a realm secret seed and mint a JWT locally (env: BOXEL_REALM_SECRET_SEED)',
+    )
+    .action(
+      async (
+        sourceCardUrl: string,
+        localDir: string,
+        options: {
+          realm?: string;
+          dryRun?: boolean;
+          realmSecretSeed?: boolean;
+        },
+      ) => {
+        let realmSecretSeed = await resolveRealmSecretSeed(
+          options.realmSecretSeed === true,
+        );
+        let result = await ingestCard(sourceCardUrl, localDir, {
+          realm: options.realm,
+          dryRun: options.dryRun,
+          realmSecretSeed,
+        });
+        if (result.error) {
+          console.error(`Error: ${result.error}`);
+          process.exit(result.files.length > 0 ? 2 : 1);
+        }
+        console.log('Ingest completed successfully');
+      },
+    );
+}