@cardstack/boxel-cli 0.1.3 → 0.2.0-unstable.294

package/src/commands/realm/{watch.ts → watch/start.ts}

@@ -1,27 +1,31 @@
 import { InvalidArgumentError, type Command } from 'commander';
 import * as fs from 'fs/promises';
 import * as path from 'path';
-import { RealmSyncBase, isProtectedFile } from '../../lib/realm-sync-base';
+import { RealmSyncBase, isProtectedFile } from '../../../lib/realm-sync-base';
 import {
   CheckpointManager,
   type Checkpoint,
   type CheckpointChange,
-} from '../../lib/checkpoint-manager';
+} from '../../../lib/checkpoint-manager';
 import {
   type SyncManifest,
   computeFileHash,
   loadManifest,
   saveManifest,
-} from '../../lib/sync-manifest';
-import type { ProfileManager } from '../../lib/profile-manager';
-import type { RealmAuthenticator } from '../../lib/realm-authenticator';
-import { resolveRealmAuthenticator } from '../../lib/auth-resolver';
-import { resolveRealmSecretSeed } from '../../lib/prompt';
+} from '../../../lib/sync-manifest';
+import type { ProfileManager } from '../../../lib/profile-manager';
+import type { RealmAuthenticator } from '../../../lib/realm-authenticator';
+import { resolveRealmAuthenticator } from '../../../lib/auth-resolver';
+import { resolveRealmSecretSeed } from '../../../lib/prompt';
 import {
   acquireWatchLock,
   releaseWatchLock,
   type WatchLockInfo,
-} from '../../lib/watch-lock';
+} from '../../../lib/watch-lock';
+import {
+  registerProcess,
+  unregisterCurrentProcess,
+} from '../../../lib/watch-process-registry';
 import {
   FG_CYAN,
   FG_GREEN,
@@ -29,7 +33,7 @@ import {
   FG_YELLOW,
   DIM,
   RESET,
-} from '../../lib/colors';
+} from '../../../lib/colors';
 
 export interface WatchRealmSpec {
   realmUrl: string;
@@ -472,6 +476,12 @@ export async function watchRealms(
     }, intervalMs);
   };
 
+  try {
+    await registerProcess(specs.map((s) => s.localDir).join(', '));
+  } catch {
+    // Best effort — registry failures must never block the watch.
+  }
+
   await tickAll();
   scheduleNextTick();
 
@@ -496,6 +506,11 @@ export async function watchRealms(
           // Best effort \u2014 a leftover lock will be detected as stale next run.
         }
       }
+      try {
+        await unregisterCurrentProcess();
+      } catch {
+        // Best effort \u2014 leftover entries are pruned on next read.
+      }
       resolve();
     };
 
@@ -572,11 +587,11 @@ function parseNonNegativeSeconds(name: string): (value: string) => number {
   };
 }
 
-export function registerWatchCommand(realm: Command): void {
-  realm
-    .command('watch')
+export function registerStartCommand(watch: Command): void {
+  watch
+    .command('start')
     .description(
-      'Watch a Boxel realm for server-side changes and pull them into a local directory',
+      'Start watching a Boxel realm for server-side changes and pull them into a local directory',
     )
     .argument(
       '<realm-url>',

package/src/commands/realm/watch/stop.ts

@@ -0,0 +1,151 @@
+import { execSync } from 'child_process';
+import type { Command } from 'commander';
+import { listRegisteredProcesses } from '../../../lib/watch-process-registry';
+import { DIM, FG_GREEN, FG_RED, RESET } from '../../../lib/colors';
+
+export interface StoppedProcess {
+  pid: number;
+  workspace: string;
+}
+
+export interface StopResult {
+  stopped: StoppedProcess[];
+  failed: StoppedProcess[];
+}
+
+const SETTLE_MS = 200;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function signalProcess(pid: number): { ok: boolean; alreadyGone: boolean } {
+  try {
+    if (process.platform === 'win32') {
+      try {
+        process.kill(pid);
+      } catch {
+        execSync(`taskkill /PID ${pid} /F`, { stdio: 'ignore' });
+      }
+    } else {
+      process.kill(pid, 'SIGINT');
+    }
+    return { ok: true, alreadyGone: false };
+  } catch (err: any) {
+    if (err?.code === 'ESRCH') {
+      return { ok: true, alreadyGone: true };
+    }
+    return { ok: false, alreadyGone: false };
+  }
+}
+
+interface PsHit {
+  pid: number;
+  workspace: string;
+}
+
+function findViaProcessTable(): PsHit[] {
+  if (process.platform === 'win32') return [];
+  let output: string;
+  try {
+    output = execSync(
+      'ps aux | grep -E "(tsx[[:space:]].*src/index\\.ts[[:space:]]+realm[[:space:]]+watch[[:space:]]+start|[[:space:]]boxel[[:space:]]+realm[[:space:]]+watch[[:space:]]+start|node[[:space:]].*boxel[[:space:]]+realm[[:space:]]+watch[[:space:]]+start)" | grep -v grep | grep -v "[[:space:]]stop"',
+      { encoding: 'utf8' },
+    ).trim();
+  } catch {
+    return [];
+  }
+  if (!output) return [];
+
+  const hits: PsHit[] = [];
+  const seen = new Set<number>();
+  for (const line of output.split('\n')) {
+    if (!line) continue;
+    const parts = line.trim().split(/\s+/);
+    const pid = Number.parseInt(parts[1] ?? '', 10);
+    if (!Number.isFinite(pid) || seen.has(pid)) continue;
+    seen.add(pid);
+
+    let workspace = '.';
+    const match = line.match(/\bstart\s+\S+\s+(\S+)/);
+    if (match && match[1] && !match[1].startsWith('-')) {
+      workspace = match[1];
+    }
+    hits.push({ pid, workspace });
+  }
+  return hits;
+}
+
+export async function stopWatchProcesses(): Promise<StopResult> {
+  const stopped: StoppedProcess[] = [];
+  const failed: StoppedProcess[] = [];
+  const targetedPids = new Set<number>();
+
+  const registered = await listRegisteredProcesses();
+  for (const proc of registered) {
+    if (proc.pid === process.pid) continue;
+    targetedPids.add(proc.pid);
+    const result = signalProcess(proc.pid);
+    const record: StoppedProcess = { pid: proc.pid, workspace: proc.workspace };
+    if (result.ok) {
+      stopped.push(record);
+    } else {
+      failed.push(record);
+    }
+  }
+
+  for (const hit of findViaProcessTable()) {
+    if (hit.pid === process.pid) continue;
+    if (targetedPids.has(hit.pid)) continue;
+    targetedPids.add(hit.pid);
+    const result = signalProcess(hit.pid);
+    const record: StoppedProcess = { pid: hit.pid, workspace: hit.workspace };
+    if (result.ok) {
+      stopped.push(record);
+    } else {
+      failed.push(record);
+    }
+  }
+
+  if (stopped.length > 0) {
+    await sleep(SETTLE_MS);
+    // Trigger another prune so the registry doesn't keep stale entries
+    // for processes that exited cleanly above.
+    await listRegisteredProcesses();
+  }
+
+  return { stopped, failed };
+}
+
+function printResult(result: StopResult): void {
+  if (result.stopped.length === 0 && result.failed.length === 0) {
+    console.log('No running watch processes found.');
+    return;
+  }
+  for (const proc of result.stopped) {
+    console.log(
+      `  ${DIM}⇅${RESET} Stopped: boxel realm watch ${proc.workspace} (PID ${proc.pid})`,
+    );
+  }
+  for (const proc of result.failed) {
+    console.log(
+      `  ${FG_RED}×${RESET} Failed to stop: boxel realm watch ${proc.workspace} (PID ${proc.pid})`,
+    );
+  }
+  if (result.stopped.length > 0) {
+    const plural = result.stopped.length > 1 ? 'es' : '';
+    console.log(
+      `\n${FG_GREEN}✓ Stopped ${result.stopped.length} process${plural}${RESET}`,
+    );
+  }
+}
+
+export function registerStopCommand(watch: Command): void {
+  watch
+    .command('stop')
+    .description('Stop all running boxel realm watch processes')
+    .action(async () => {
+      const result = await stopWatchProcesses();
+      printResult(result);
+    });
+}

package/src/lib/boxel-cli-client.ts

@@ -450,6 +450,18 @@ export class BoxelCLIClient {
     return this.pm.authedRealmServerFetch(input, init);
   }
 
+  /**
+   * Return the realm-server JWT, fetching one via Matrix login if no token
+   * is cached. Use only when you need to hand the bare token to a downstream
+   * client that can't go through `authedServerFetch` (e.g. opencode's
+   * static-Authorization provider config). Prefer `authedServerFetch` for
+   * server endpoints called from JS — it handles per-request 401 retries
+   * that this getter cannot.
+   */
+  async getServerToken(): Promise<string> {
+    return this.pm.getOrRefreshServerToken();
+  }
+
   async pull(
     realmUrl: string,
     localDir: string,

package/src/lib/checkpoint-manager.ts

@@ -364,41 +364,46 @@ export class CheckpointManager {
       });
 
     return Promise.all(
-      lines.map(async (line) => {
-        const [hash, shortHash, subject, dateStr] = line.split('|');
-
-        const isMajor = subject.includes('[MAJOR]');
-        const source = subject.includes('[local]')
-          ? ('local' as const)
-          : subject.includes('[remote]')
-            ? ('remote' as const)
-            : ('manual' as const);
-
-        const message = subject
-          .replace(/\[(MAJOR|minor)\]\s*/i, '')
-          .replace(/\[(local|remote|manual)\]\s*/i, '');
-
-        const stats = await this.getCommitStats(hash);
-
-        const milestoneName = milestones.get(hash);
-        const isMilestone = !!milestoneName;
-
-        return {
-          hash,
-          shortHash,
-          message,
-          description: '',
-          date: new Date(dateStr),
-          isMajor,
-          source,
-          isMilestone,
-          milestoneName,
-          ...stats,
-        };
-      }),
+      lines.map((line) => this.parseCheckpointLine(line, milestones)),
     );
   }
 
+  private async parseCheckpointLine(
+    line: string,
+    milestones: Map<string, string>,
+  ): Promise<Checkpoint> {
+    const [hash, shortHash, subject, dateStr] = line.split('|');
+
+    const isMajor = subject.includes('[MAJOR]');
+    const source = subject.includes('[local]')
+      ? ('local' as const)
+      : subject.includes('[remote]')
+        ? ('remote' as const)
+        : ('manual' as const);
+
+    const message = subject
+      .replace(/\[(MAJOR|minor)\]\s*/i, '')
+      .replace(/\[(local|remote|manual)\]\s*/i, '');
+
+    const stats = await this.getCommitStats(hash);
+
+    const milestoneName = milestones.get(hash);
+    const isMilestone = !!milestoneName;
+
+    return {
+      hash,
+      shortHash,
+      message,
+      description: '',
+      date: new Date(dateStr),
+      isMajor,
+      source,
+      isMilestone,
+      milestoneName,
+      ...stats,
+    };
+  }
+
   private async getCommitStats(hash: string): Promise<{
     filesChanged: number;
     insertions: number;
@@ -575,8 +580,36 @@ export class CheckpointManager {
   }
 
   async getMilestones(): Promise<Checkpoint[]> {
-    const all = await this.getCheckpoints(100);
-    return all.filter((cp) => cp.isMilestone);
+    if (!(await this.isInitialized())) {
+      return [];
+    }
+    const milestones = await this.getAllMilestones();
+    if (milestones.size === 0) {
+      return [];
+    }
+
+    // Enumerate from the milestone tags directly so the result is complete
+    // regardless of how deep the tagged checkpoints sit in history. `--no-walk`
+    // limits `git log` to just the listed commits — no traversal, no implicit
+    // cap.
+    const format = '%H|%h|%s|%aI|%an';
+    const log = await this.git(
+      'log',
+      '--no-walk',
+      `--format=${format}`,
+      ...milestones.keys(),
+    );
+
+    if (!log.trim()) {
+      return [];
+    }
+
+    return Promise.all(
+      log
+        .trim()
+        .split('\n')
+        .map((line) => this.parseCheckpointLine(line, milestones)),
+    );
   }
 
   private git(...args: string[]): Promise<string> {

package/src/lib/profile-manager.ts

@@ -1,6 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
+import jwt from 'jsonwebtoken';
 import { FG_YELLOW, FG_CYAN, FG_MAGENTA, DIM, BOLD, RESET } from './colors';
 import {
   matrixLogin,
@@ -16,6 +17,31 @@ import type { RealmAuthenticator } from './realm-authenticator';
 const DEFAULT_CONFIG_DIR = path.join(os.homedir(), '.boxel-cli');
 const PROFILES_FILENAME = 'profiles.json';
 
+/**
+ * Tokens issued by the realm server carry a 7-day TTL. Re-mint when
+ * there's less than a day left so a long-running operation (or a
+ * downstream consumer that bakes the token into a static header, like
+ * opencode's passthrough provider) doesn't get a 401 mid-flight.
+ *
+ * Decode-only — we don't verify the signature; the realm server does
+ * that on every request. We only care about the `exp` claim.
+ */
+const SERVER_TOKEN_EXPIRY_SAFETY_MARGIN_SEC = 86400; // 1 day
+
+function isJwtNearExpiry(
+  token: string,
+  safetyMarginSec = SERVER_TOKEN_EXPIRY_SAFETY_MARGIN_SEC,
+): boolean {
+  // Tokens are cached verbatim from the realm server's `Authorization`
+  // response header, so they're prefixed with `Bearer ` — strip it before
+  // decoding or jsonwebtoken returns null and we'd refresh on every call.
+  let raw = token.replace(/^Bearer\s+/i, '');
+  let decoded = jwt.decode(raw) as { exp?: number } | null;
+  if (!decoded?.exp) return true; // unparseable / missing exp → treat as expired
+  let nowSec = Math.floor(Date.now() / 1000);
+  return decoded.exp - nowSec < safetyMarginSec;
+}
+
 export const NO_ACTIVE_PROFILE_ERROR =
   'No active profile. Run `boxel profile add` to create one.';
 
@@ -371,7 +397,7 @@ export class ProfileManager implements RealmAuthenticator {
 
   async getOrRefreshServerToken(): Promise<string> {
     let cached = this.getRealmServerToken();
-    if (cached) {
+    if (cached && !isJwtNearExpiry(cached)) {
       return cached;
     }
     let matrixAuth = await this.loginToMatrix();

package/src/lib/realm-local-paths.ts

@@ -0,0 +1,243 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { isQuiet } from './cli-log';
+
+export interface MisplacedLocalRealmEntry {
+  manifestPath: string;
+  currentDir: string;
+  expectedDir: string;
+  realmUrl: string;
+}
+
+interface MinimalManifest {
+  realmUrl: string;
+}
+
+let didWarnInProcess = false;
+
+const SKIPPABLE_DIR_NAMES = new Set([
+  '.git',
+  'node_modules',
+  'dist',
+  '.boxel-history',
+  '.claude',
+]);
+
+function isSkippableDir(dirName: string): boolean {
+  return SKIPPABLE_DIR_NAMES.has(dirName);
+}
+
+function canonicalDomainFromHost(hostname: string): string {
+  if (hostname === 'stack.cards' || hostname.endsWith('.stack.cards')) {
+    return 'stack.cards';
+  }
+  if (hostname === 'boxel.ai' || hostname.endsWith('.boxel.ai')) {
+    return 'boxel.ai';
+  }
+  return hostname;
+}
+
+// Reject segments that would let a crafted realmUrl escape the rootDir tree
+// (`.`, `..`, anything containing a path separator or NUL) — defence in depth
+// against malformed manifests; `findMisplacedLocalRealmDirs` also re-checks
+// containment after `path.resolve` collapses any `..` it didn't catch.
+function isSafePathSegment(segment: string): boolean {
+  if (!segment) return false;
+  let decoded: string;
+  try {
+    decoded = decodeURIComponent(segment);
+  } catch {
+    return false;
+  }
+  if (decoded === '.' || decoded === '..') return false;
+  if (
+    decoded.includes('/') ||
+    decoded.includes('\\') ||
+    decoded.includes('\0')
+  ) {
+    return false;
+  }
+  return true;
+}
+
+export function relativeStructuredPathForRealmUrl(
+  realmUrl: string,
+): string | null {
+  let url: URL;
+  try {
+    url = new URL(realmUrl);
+  } catch {
+    return null;
+  }
+  const domain = canonicalDomainFromHost(url.hostname);
+  if (!isSafePathSegment(domain)) return null;
+  const parts = url.pathname
+    .replace(/^\/|\/$/g, '')
+    .split('/')
+    .filter(Boolean);
+  const owner = parts[0] ?? 'unknown-owner';
+  const realm = parts[1] ?? parts[0] ?? 'workspace';
+  if (!isSafePathSegment(owner) || !isSafePathSegment(realm)) return null;
+  return path.join(domain, owner, realm);
+}
+
+export function absoluteStructuredPathForRealmUrl(
+  realmUrl: string,
+  rootDir: string,
+): string | null {
+  const rel = relativeStructuredPathForRealmUrl(realmUrl);
+  if (rel === null) return null;
+  return path.resolve(rootDir, rel);
+}
+
+function tryReadRealmUrl(manifestPath: string): MinimalManifest | null {
+  let content: string;
+  try {
+    content = fs.readFileSync(manifestPath, 'utf-8');
+  } catch {
+    return null;
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return null;
+  }
+  if (typeof parsed !== 'object' || parsed === null) {
+    return null;
+  }
+  const candidate = (parsed as Record<string, unknown>).realmUrl;
+  if (typeof candidate !== 'string' || candidate === '') {
+    return null;
+  }
+  return { realmUrl: candidate };
+}
+
+function addManifestIfExists(dir: string, manifests: string[]): void {
+  const manifestPath = path.join(dir, '.boxel-sync.json');
+  if (fs.existsSync(manifestPath)) {
+    manifests.push(manifestPath);
+  }
+}
+
+function listSubdirs(dir: string): string[] {
+  try {
+    return fs
+      .readdirSync(dir, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory() && !isSkippableDir(entry.name))
+      .map((entry) => path.join(dir, entry.name));
+  } catch {
+    return [];
+  }
+}
+
+function findManifestPaths(rootDir: string): string[] {
+  const manifests: string[] = [];
+  const absoluteRoot = path.resolve(rootDir);
+
+  // Legacy layout: <root>/<realm>/.boxel-sync.json
+  for (const childDir of listSubdirs(absoluteRoot)) {
+    addManifestIfExists(childDir, manifests);
+  }
+
+  // Canonical layout: <root>/<domain>/<owner>/<realm>/.boxel-sync.json
+  for (const domainDir of listSubdirs(absoluteRoot)) {
+    for (const ownerDir of listSubdirs(domainDir)) {
+      for (const realmDir of listSubdirs(ownerDir)) {
+        addManifestIfExists(realmDir, manifests);
+      }
+    }
+  }
+
+  return manifests;
+}
+
+// True iff `child` is `root` or a descendant of `root`. Belt-and-suspenders
+// containment check after `path.resolve` — even if a crafted realmUrl made
+// it past `isSafePathSegment`, the resolved path must stay inside rootDir
+// before we move anything.
+function isWithin(root: string, child: string): boolean {
+  const rel = path.relative(root, child);
+  if (rel === '') return true;
+  if (rel.startsWith('..')) return false;
+  return !path.isAbsolute(rel);
+}
+
+export function findMisplacedLocalRealmDirs(
+  rootDir: string,
+): MisplacedLocalRealmEntry[] {
+  const absoluteRoot = path.resolve(rootDir);
+  const manifestPaths = findManifestPaths(absoluteRoot);
+
+  const seenManifestPaths = new Set<string>();
+  const entries: MisplacedLocalRealmEntry[] = [];
+  for (const manifestPath of manifestPaths) {
+    if (seenManifestPaths.has(manifestPath)) {
+      continue;
+    }
+    seenManifestPaths.add(manifestPath);
+
+    const manifest = tryReadRealmUrl(manifestPath);
+    if (!manifest) {
+      continue;
+    }
+
+    const expectedDir = absoluteStructuredPathForRealmUrl(
+      manifest.realmUrl,
+      absoluteRoot,
+    );
+    if (expectedDir === null) {
+      continue;
+    }
+    if (!isWithin(absoluteRoot, expectedDir)) {
+      continue;
+    }
+
+    const currentDir = path.dirname(manifestPath);
+    if (path.resolve(currentDir) !== path.resolve(expectedDir)) {
+      entries.push({
+        manifestPath,
+        currentDir,
+        expectedDir,
+        realmUrl: manifest.realmUrl,
+      });
+    }
+  }
+
+  return entries;
+}
+
+export function warnIfMisplacedLocalRealmDirs(rootDir: string): void {
+  if (didWarnInProcess) return;
+  if (process.env.BOXEL_DISABLE_PATH_WARNING === '1') return;
+  if (isQuiet()) return;
+
+  const entries = findMisplacedLocalRealmDirs(rootDir);
+  if (entries.length === 0) return;
+
+  didWarnInProcess = true;
+
+  console.warn('\n⚠️  Detected local realm directories at legacy local paths:');
+  const absoluteRoot = path.resolve(rootDir);
+  for (const entry of entries.slice(0, 5)) {
+    const from = path.relative(absoluteRoot, entry.currentDir) || '.';
+    const to = path.relative(absoluteRoot, entry.expectedDir) || '.';
+    console.warn(`   - ${from} -> ${to}`);
+  }
+  if (entries.length > 5) {
+    console.warn(`   ...and ${entries.length - 5} more`);
+  }
+  console.warn('\nRun to preview:');
+  console.warn('   boxel consolidate-workspaces . --dry-run');
+  console.warn('Then apply:');
+  console.warn('   boxel consolidate-workspaces .\n');
+}
+
+/**
+ * Test-only escape hatch — resets the once-per-process warning latch so tests
+ * can exercise `warnIfMisplacedLocalRealmDirs` repeatedly within a single
+ * Node process.
+ */
+export function resetWarnedFlagForTests(): void {
+  didWarnInProcess = false;
+}

package/src/lib/realm-sync-base.ts

@@ -125,11 +125,13 @@ export abstract class RealmSyncBase {
     try {
       const url = this.buildDirectoryUrl(dir);
 
-      const response = await this.authenticator.authedRealmFetch(url, {
-        headers: {
-          Accept: 'application/vnd.api+json',
-        },
-      });
+      const response = await this.remoteLimit(() =>
+        this.authenticator.authedRealmFetch(url, {
+          headers: {
+            Accept: 'application/vnd.api+json',
+          },
+        }),
+      );
 
       if (!response.ok) {
         if (response.status === 404) {
@@ -167,10 +169,10 @@ export abstract class RealmSyncBase {
               }
               return [] as Array<[string, boolean]>;
             } else {
-              return this.remoteLimit(async () => {
+              return (async () => {
                 const subdirFiles = await this.getRemoteFileList(entryPath);
                 return Array.from(subdirFiles.entries());
-              });
+              })();
             }
           }),
         );