@celilo/cli 0.3.30-alpha.0 → 0.4.0-alpha.0

package/src/cli/commands/publish/workspace.ts

@@ -0,0 +1,596 @@
+/**
+ * Workspace publish — Phase 1 of `celilo publish`.
+ *
+ * Planner/executor split per v2/PUBLILO_CLI.md Phase 2:
+ *
+ *   planWorkspace(opts)  →  WorkspaceItem[]   // pure, no mutations
+ *   executeWorkspace(plan, ...)               // mutates package.json,
+ *                                             // runs bun publish,
+ *                                             // restores, verifies
+ *
+ * The planner figures out which versions each package would publish at
+ * (real, alpha-N, promoted-base), which workspace:^ deps need explicit
+ * rewriting, whether the package should be skipped (already on npm,
+ * source unchanged since prior alpha), and which pre-publish hooks
+ * fire. The executor applies the plan top-to-bottom — no decisions are
+ * made during execution, just side effects.
+ */
+
+import { spawnSync } from 'node:child_process';
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { join, relative } from 'node:path';
+import { ALPHA_TAG, alphaSkipDecision, nextAlphaNumber, stripAlphaSuffix } from './alpha';
+import { REPO_ROOT, isPublished, readPkg } from './helpers';
+import type {
+  PackageJson,
+  PublishMode,
+  PublishResult,
+  RewriteOptions,
+  WorkspaceItem,
+  WorkspaceRewrite,
+  WorkspaceVersionMap,
+} from './types';
+
+// ─── Planner ───────────────────────────────────────────────────────
+
+export interface PlanWorkspaceInput {
+  mode: PublishMode;
+  packages: readonly string[];
+  /**
+   * Starting workspace-version map (typically built from each
+   * package.json's `version` field). The planner mutates a local copy
+   * — in alpha mode it tightens the map to the alpha versions so
+   * sibling rewrites resolve correctly. Doesn't mutate the input.
+   */
+  baseWorkspaceVersions: WorkspaceVersionMap;
+  gitHead: string;
+}
+
+export interface PlanWorkspaceOutput {
+  items: WorkspaceItem[];
+  /**
+   * The map of name → versionToPublish for non-skipped items. The
+   * executor uses this to rewrite workspace:^ deps consistently across
+   * sibling packages in the same publish run.
+   */
+  workspaceVersions: WorkspaceVersionMap;
+}
+
+/**
+ * Pre-publish hooks that fire for @celilo/e2e (the only package that
+ * needs them today). Listed in the WorkspaceItem so dry-run can show
+ * "@celilo/e2e (will rebuild netapps, stage caches)" without executing
+ * anything.
+ */
+function workspaceHooksFor(pkg: string): WorkspaceItem['hooks'] {
+  if (pkg !== 'packages/e2e') return [];
+  return ['registryServerBundle', 'rebuildE2eNetapps', 'stageE2ePublishCaches'];
+}
+
+/**
+ * Build the workspace publish plan. Pure given the world-state reads
+ * exposed by helpers (npm view, git log) — same world → same plan.
+ *
+ * In promote mode, only the named package is planned; everything else
+ * is filtered out. In alpha mode, the local workspaceVersions map gets
+ * tightened to alpha versions as we iterate, so later packages in
+ * dependency order can pin to the alpha version of earlier siblings.
+ */
+export function planWorkspace(opts: PlanWorkspaceInput): PlanWorkspaceOutput {
+  const { mode, packages, baseWorkspaceVersions } = opts;
+  const workspaceVersions = new Map(baseWorkspaceVersions);
+
+  const packagesToPlan: readonly string[] =
+    mode.kind === 'promote'
+      ? packages.filter((p) => readPkg(p).name === mode.target.name)
+      : packages;
+
+  if (mode.kind === 'promote' && packagesToPlan.length === 0) {
+    throw new Error(
+      `--promote target "${mode.target.name}" is not a known workspace package. Known packages: ${[
+        ...workspaceVersions.keys(),
+      ].join(', ')}`,
+    );
+  }
+
+  const items: WorkspaceItem[] = [];
+  for (const pkg of packagesToPlan) {
+    const { name, version } = readPkg(pkg);
+    if (!name || !version) {
+      throw new Error(`${pkg}/package.json missing name or version`);
+    }
+
+    let versionToPublish: string;
+    let skipReason: string | undefined;
+    let tag: WorkspaceItem['tag'];
+
+    if (mode.kind === 'promote') {
+      versionToPublish = stripAlphaSuffix(mode.target.version);
+      if (versionToPublish === mode.target.version) {
+        throw new Error(
+          `--promote target "${mode.target.name}@${mode.target.version}" is not an alpha (no -alpha.N suffix).`,
+        );
+      }
+    } else if (mode.kind === 'alpha') {
+      const n = nextAlphaNumber(name, version);
+      const decision = alphaSkipDecision(pkg, name, version, n);
+      if (decision.skip) {
+        // Skipped (unchanged since its last alpha): it will NOT be
+        // republished, so consumers must pin to the alpha that already
+        // exists on npm (alpha.{n-1}), not the computed-next alpha.{n}
+        // which would never be published. (n >= 1 whenever skip is true —
+        // nextN === 0 can't skip.) Pinning to alpha.{n} made dependents
+        // unpublishable: "@celilo/cli-display@0.1.9-alpha.1 is not on npm".
+        skipReason = decision.reason;
+        versionToPublish = `${version}-alpha.${n - 1}`;
+      } else {
+        versionToPublish = `${version}-alpha.${n}`;
+      }
+      tag = ALPHA_TAG;
+      workspaceVersions.set(name, versionToPublish);
+    } else {
+      versionToPublish = version;
+      if (isPublished(name, versionToPublish)) {
+        skipReason = 'already published';
+      }
+    }
+
+    items.push({
+      pkg,
+      name,
+      baseVersion: version,
+      versionToPublish,
+      tag,
+      rewriteOptions: {
+        targetVersion: versionToPublish !== version ? versionToPublish : undefined,
+        exactPins: mode.kind === 'alpha',
+        gitHead: mode.kind === 'alpha' ? opts.gitHead : undefined,
+      },
+      hooks: workspaceHooksFor(pkg),
+      skipReason,
+    });
+  }
+
+  return { items, workspaceVersions };
+}
+
+// ─── Write-side helpers (used by executor) ─────────────────────────
+
+/**
+ * Bun's publish-time rewrite of `workspace:^` is unreliable — empirically,
+ * it's been baking in `^0.1.0` regardless of the actual current version
+ * of the dep. We rewrite explicitly here:
+ *   workspace:^   → ^<current-workspace-version>     (or exact, in alpha mode)
+ *   workspace:~   → ~<current-workspace-version>     (or exact, in alpha mode)
+ *   workspace:*   → * (passes through; means "any version")
+ *   workspace:X.Y.Z → X.Y.Z (explicit; passes through)
+ *
+ * Returns the original package.json content so the caller can restore
+ * after `bun publish` runs. We don't want to commit the rewritten form
+ * — `workspace:^` in source is more readable for dev.
+ */
+export function rewriteWorkspaceDeps(
+  pkg: string,
+  versions: WorkspaceVersionMap,
+  opts: RewriteOptions = {},
+  /**
+   * Optional override for the repo root. Production code uses the
+   * module-resolved REPO_ROOT (the real monorepo); tests can hand in
+   * a synthetic workspace path so the rewrite hits temp-dir files.
+   */
+  repoRoot: string = REPO_ROOT,
+): { original: string; rewrites: WorkspaceRewrite[] } {
+  const path = join(repoRoot, pkg, 'package.json');
+  const original = readFileSync(path, 'utf-8');
+  const parsed = JSON.parse(original) as PackageJson & { gitHead?: string };
+  const rewrites: WorkspaceRewrite[] = [];
+
+  let mutated = false;
+  if (opts.targetVersion !== undefined && opts.targetVersion !== parsed.version) {
+    parsed.version = opts.targetVersion;
+    mutated = true;
+  }
+  if (opts.gitHead !== undefined && opts.gitHead !== parsed.gitHead) {
+    parsed.gitHead = opts.gitHead;
+    mutated = true;
+  }
+
+  const buckets: Array<keyof PackageJson> = [
+    'dependencies',
+    'devDependencies',
+    'peerDependencies',
+    'optionalDependencies',
+  ];
+  for (const bucket of buckets) {
+    const deps = parsed[bucket] as Record<string, string> | undefined;
+    if (!deps) continue;
+    for (const [name, oldSpec] of Object.entries(deps)) {
+      if (!oldSpec.startsWith('workspace:')) continue;
+      const wsRange = oldSpec.slice('workspace:'.length);
+      const targetVersion = versions.get(name);
+      if (!targetVersion) {
+        continue;
+      }
+      let newSpec: string;
+      if (wsRange === '*' || wsRange === '') {
+        newSpec = '*';
+      } else if (opts.exactPins) {
+        newSpec = targetVersion;
+      } else if (wsRange === '^') {
+        newSpec = `^${targetVersion}`;
+      } else if (wsRange === '~') {
+        newSpec = `~${targetVersion}`;
+      } else {
+        newSpec = wsRange;
+      }
+      deps[name] = newSpec;
+      rewrites.push({ depName: name, bucket: String(bucket), oldSpec, newSpec });
+    }
+  }
+  if (rewrites.length > 0 || mutated) {
+    const trailingNl = original.endsWith('\n') ? '\n' : '';
+    writeFileSync(path, JSON.stringify(parsed, null, 2) + trailingNl);
+  }
+  return { original, rewrites };
+}
+
+export function restorePackageJson(
+  pkg: string,
+  original: string,
+  repoRoot: string = REPO_ROOT,
+): void {
+  writeFileSync(join(repoRoot, pkg, 'package.json'), original);
+}
+
+/**
+ * After `bun publish` succeeds, ask npm what dep versions the freshly-
+ * published package.json contains. If any rewrite we made didn't make
+ * it through to the published artifact, fail loudly — don't silently
+ * leave a broken pin on npm.
+ */
+export async function verifyPublishedDeps(
+  name: string,
+  version: string,
+  rewrites: WorkspaceRewrite[],
+  /**
+   * Optional override pointing `npm view` at a different registry.
+   * Production omits this (uses the operator's configured npm/bun
+   * registry); tests pass their verdaccio URL.
+   */
+  registryUrl?: string,
+): Promise<void> {
+  // npm's registry replication is eventually-consistent. A package
+  // published 1s ago can still return empty for `npm view ... dep.X`
+  // even though the publish succeeded. Retry with backoff before
+  // declaring failure — only fail if the pin is genuinely WRONG, not
+  // just temporarily missing.
+  for (const r of rewrites) {
+    let actualPin = '';
+    let attempt = 0;
+    const maxAttempts = 6;
+    const delays = [500, 1000, 2000, 3000, 5000, 8000];
+    const baseArgs = ['view', `${name}@${version}`, `${r.bucket}.${r.depName}`];
+    const args = registryUrl ? [...baseArgs, '--registry', registryUrl] : baseArgs;
+    while (attempt < maxAttempts) {
+      const result = spawnSync('npm', args, {
+        stdio: ['ignore', 'pipe', 'pipe'],
+        encoding: 'utf-8',
+      });
+      actualPin = (result.stdout ?? '').trim();
+      if (actualPin) break;
+      attempt++;
+      if (attempt < maxAttempts) {
+        const delay = delays[attempt - 1];
+        process.stdout.write(
+          `  (verify retry ${attempt} for ${r.depName} — npm view returned empty, waiting ${delay}ms…)\n`,
+        );
+        await new Promise((r2) => setTimeout(r2, delay));
+      }
+    }
+    if (!actualPin) {
+      console.warn(
+        `⚠ verify could not read ${name}@${version} ${r.bucket}.${r.depName} from npm after ${maxAttempts} retries. Skipping verify for this dep.`,
+      );
+      continue;
+    }
+    if (actualPin !== r.newSpec) {
+      console.error(
+        `✗ ${name}@${version} published with ${r.depName} pinned at "${actualPin}", expected "${r.newSpec}"`,
+      );
+      console.error(
+        '  The pre-publish workspace:^ rewrite either failed silently or\n' +
+          '  bun stripped our edit. Inspect the published artifact and the\n' +
+          '  source package.json. The package is on npm with the wrong pin —\n' +
+          '  consumers will get a stale dep until you republish.',
+      );
+      process.exit(1);
+    }
+  }
+}
+
+// ─── Per-package pre-publish hooks ─────────────────────────────────
+
+/**
+ * Modules excluded from the @celilo/e2e netapps shipment:
+ *   - celilo-registry: bundles the bun-compiled registry server
+ *     binaries (~76 MB packed). Not used by typical consumer e2e
+ *     tests; including it would bloat the npm tarball past the SSL
+ *     transport's reliable window.
+ *   - archive: not a real module dir.
+ */
+const E2E_NETAPP_EXCLUDES = new Set(['archive', 'celilo-registry']);
+
+/**
+ * Pre-publish step for @celilo/e2e: package each (non-excluded)
+ * module under `<root>/modules/` into `packages/e2e/netapps/`. Replaces
+ * whatever was last left there by a local `cele2e build-infra` so the
+ * shipped tarball is always built from the current branch, not the
+ * publisher's last local development build.
+ */
+export function rebuildE2eNetapps(repoRoot: string): void {
+  const modulesDir = join(repoRoot, 'modules');
+  const netappsDir = join(repoRoot, 'packages/e2e/netapps');
+  const celiloWrapper = join(repoRoot, 'celilo');
+
+  if (!existsSync(modulesDir)) {
+    console.warn(
+      `⚠ ${modulesDir} not found — skipping netapp rebuild. The published tarball will reuse whatever's currently in packages/e2e/netapps/.`,
+    );
+    return;
+  }
+  if (!existsSync(celiloWrapper)) {
+    console.warn(
+      `⚠ ${celiloWrapper} not found — skipping netapp rebuild. Same staleness risk as above.`,
+    );
+    return;
+  }
+
+  console.log('Rebuilding packages/e2e/netapps/ from infra/modules/...');
+
+  for (const existing of readdirSync(netappsDir).filter((f) => f.endsWith('.netapp'))) {
+    try {
+      unlinkSync(join(netappsDir, existing));
+    } catch {
+      // Best effort
+    }
+  }
+
+  const moduleDirs = readdirSync(modulesDir).filter((name) => {
+    if (E2E_NETAPP_EXCLUDES.has(name)) return false;
+    const dir = join(modulesDir, name);
+    try {
+      return statSync(dir).isDirectory() && existsSync(join(dir, 'manifest.yml'));
+    } catch {
+      return false;
+    }
+  });
+
+  let okCount = 0;
+  const failures: Array<{ name: string; stderr: string }> = [];
+  for (const name of moduleDirs) {
+    const moduleDir = join(modulesDir, name);
+    const out = join(netappsDir, `${name}.netapp`);
+    const r = spawnSync(celiloWrapper, ['package', moduleDir, '--output', out], {
+      cwd: repoRoot,
+      stdio: ['ignore', 'pipe', 'pipe'],
+      encoding: 'utf-8',
+    });
+    if (r.status === 0) {
+      okCount++;
+    } else {
+      failures.push({ name, stderr: (r.stderr ?? '').trim() || `exit ${r.status}` });
+    }
+  }
+
+  if (failures.length > 0) {
+    console.error(`✗ Failed to package ${failures.length} module(s):`);
+    for (const f of failures) {
+      console.error(`  ${f.name}: ${f.stderr}`);
+    }
+    console.error(
+      'Aborting publish — shipping with missing netapps would silently break consumer e2e tests.',
+    );
+    process.exit(1);
+  }
+
+  console.log(
+    `Refreshed ${okCount} netapp(s) in packages/e2e/netapps/ (excluded: ${[...E2E_NETAPP_EXCLUDES].join(', ')}).\n`,
+  );
+}
+
+/**
+ * Pre-publish step for @celilo/e2e: stage `.celilo-website-cache/` and
+ * `.npm-registry-cache/` inside packages/e2e/ so the tarball ships with
+ * everything Dockerfile.celilo-website-sim and Dockerfile.npm-registry-sim
+ * need to COPY at docker-build time.
+ */
+export function stageE2ePublishCaches(repoRoot: string): void {
+  const websiteSrc = join(repoRoot, 'modules', 'celilo-website', 'site');
+  const e2eDir = join(repoRoot, 'packages', 'e2e');
+  const websiteCache = join(e2eDir, '.celilo-website-cache');
+  const npmCache = join(e2eDir, '.npm-registry-cache');
+  const packScript = join(e2eDir, 'scripts', 'pack-celilo-packages.ts');
+
+  if (!existsSync(websiteSrc)) {
+    console.error(`✗ ${websiteSrc} not found — cannot stage .celilo-website-cache.`);
+    process.exit(1);
+  }
+  if (!existsSync(packScript)) {
+    console.error(`✗ ${packScript} not found — cannot stage .npm-registry-cache.`);
+    process.exit(1);
+  }
+
+  console.log('Staging .celilo-website-cache/ from modules/celilo-website/site/...');
+  const installResult = spawnSync('bun', ['install'], { cwd: websiteSrc, stdio: 'pipe' });
+  if (installResult.status !== 0) {
+    console.error('✗ bun install for celilo-website failed:');
+    console.error(installResult.stderr?.toString());
+    process.exit(1);
+  }
+  const buildResult = spawnSync('bun', ['run', 'build'], { cwd: websiteSrc, stdio: 'pipe' });
+  if (buildResult.status !== 0) {
+    console.error('✗ bun run build for celilo-website failed:');
+    console.error(buildResult.stderr?.toString());
+    process.exit(1);
+  }
+  rmSync(websiteCache, { recursive: true, force: true });
+  mkdirSync(websiteCache, { recursive: true });
+  cpSync(join(websiteSrc, 'dist'), websiteCache, { recursive: true });
+  console.log(`✓ Staged .celilo-website-cache/ (from ${relative(repoRoot, websiteSrc)}/dist/)\n`);
+
+  console.log('Staging .npm-registry-cache/ from @celilo/* workspace tarballs...');
+  const packResult = spawnSync('bun', ['run', packScript], { cwd: repoRoot, stdio: 'pipe' });
+  if (packResult.status !== 0) {
+    console.error('✗ pack-celilo-packages.ts failed:');
+    console.error(packResult.stderr?.toString());
+    process.exit(1);
+  }
+  const packed = existsSync(npmCache)
+    ? readdirSync(npmCache).filter((f) => f.endsWith('.tgz'))
+    : [];
+  console.log(`✓ Staged .npm-registry-cache/ with ${packed.length} workspace tarball(s)\n`);
+}
+
+// ─── Executor ──────────────────────────────────────────────────────
+
+export interface ExecuteWorkspaceInput {
+  items: WorkspaceItem[];
+  workspaceVersions: WorkspaceVersionMap;
+  mode: PublishMode;
+  /** From the confirm() helper in index.ts — passed in to avoid coupling. */
+  confirm: (question: string) => Promise<boolean>;
+}
+
+export async function executeWorkspace(input: ExecuteWorkspaceInput): Promise<PublishResult> {
+  const { items, workspaceVersions, mode, confirm } = input;
+  const published: PublishResult['published'] = [];
+  const skipped: string[] = [];
+
+  for (const item of items) {
+    const { pkg, name, baseVersion, versionToPublish } = item;
+
+    if (item.skipReason) {
+      console.log(`\n→ ${name}@${versionToPublish} skipped (${item.skipReason}).`);
+      skipped.push(`${name}@${versionToPublish} (${item.skipReason})`);
+      continue;
+    }
+
+    console.log('\n──────────────────────────────────────────────');
+    const tag = mode.kind === 'alpha' ? '  [alpha]' : mode.kind === 'promote' ? '  [promote]' : '';
+    console.log(` ${name}@${versionToPublish}  (${pkg})${tag}`);
+    console.log('──────────────────────────────────────────────\n');
+
+    // Per-package pre-publish hooks. Driven off the WorkspaceItem so the
+    // dry-run plan listed them too.
+    if (item.hooks.includes('registryServerBundle')) {
+      const { ensureRegistryServerBundle } = await import(
+        join(REPO_ROOT, 'packages/e2e/src/registry-bundle.ts')
+      );
+      ensureRegistryServerBundle(join(REPO_ROOT, 'packages/e2e'));
+      console.log(
+        'Refreshed packages/e2e/registry-server/ bundle from packages/registry-server.\n',
+      );
+    }
+    if (item.hooks.includes('rebuildE2eNetapps')) {
+      rebuildE2eNetapps(REPO_ROOT);
+    }
+    if (item.hooks.includes('stageE2ePublishCaches')) {
+      stageE2ePublishCaches(REPO_ROOT);
+    }
+
+    const { original: pkgJsonOriginal, rewrites: workspaceRewrites } = rewriteWorkspaceDeps(
+      pkg,
+      workspaceVersions,
+      item.rewriteOptions,
+    );
+    if (workspaceRewrites.length > 0) {
+      console.log('Rewrote workspace deps to explicit versions:');
+      for (const r of workspaceRewrites) {
+        console.log(`  ${r.bucket}.${r.depName}: ${r.oldSpec} → ${r.newSpec}`);
+      }
+      console.log();
+    }
+
+    // Belt-and-suspenders: refuse to publish if any @celilo/* dep
+    // (after workspace-rewrite) doesn't exist on npm. We publish in dep
+    // order, so this should be a no-op — but it catches operator typos
+    // and any external `bun publish` invocation that bypasses the
+    // ordered loop.
+    const checkResult = spawnSync('bun', [join(REPO_ROOT, 'scripts/check-publishable.ts'), pkg], {
+      cwd: REPO_ROOT,
+      stdio: 'inherit',
+    });
+    if (checkResult.status !== 0) {
+      restorePackageJson(pkg, pkgJsonOriginal);
+      console.error(
+        `✗ Refusing to publish ${name}@${versionToPublish} — see check-publishable output above.`,
+      );
+      process.exit(1);
+    }
+
+    const publishArgs = ['publish', '--access', 'public'];
+    if (item.tag) publishArgs.push('--tag', item.tag);
+
+    let publishStatus: number | null = null;
+    let publishError: unknown = null;
+    try {
+      spawnSync('bun', [...publishArgs, '--dry-run'], {
+        cwd: join(REPO_ROOT, pkg),
+        stdio: 'inherit',
+      });
+
+      if (!(await confirm(`\nPublish ${name}@${versionToPublish} for real? [y/N] `))) {
+        console.log('Skipped.');
+        skipped.push(`${name}@${versionToPublish} (manual skip)`);
+        continue;
+      }
+
+      const r = spawnSync('bun', publishArgs, { cwd: join(REPO_ROOT, pkg), stdio: 'inherit' });
+      publishStatus = r.status;
+    } catch (err) {
+      publishError = err;
+    } finally {
+      // ALWAYS restore the source package.json so workspace:^ stays in
+      // git. Even on publish failure or operator skip.
+      restorePackageJson(pkg, pkgJsonOriginal);
+    }
+
+    if (publishError) throw publishError;
+
+    if (publishStatus !== 0) {
+      // npm view can be transiently stale — recheck once before failing.
+      if (isPublished(name, versionToPublish)) {
+        console.log(
+          `\n→ ${name}@${versionToPublish} reports already-published on recheck (initial isPublished hit a stale npm view). Skipping.`,
+        );
+        skipped.push(`${name}@${versionToPublish} (already published — detected on retry)`);
+        continue;
+      }
+      console.error(`✗ Publish failed for ${name}@${versionToPublish}`);
+      process.exit(publishStatus ?? 1);
+    }
+
+    if (workspaceRewrites.length > 0) {
+      await verifyPublishedDeps(name, versionToPublish, workspaceRewrites);
+    }
+
+    // Silence the unused-var warning on baseVersion — we keep the value
+    // on the WorkspaceItem for dry-run output ("0.7.13 → 0.7.13-alpha.0").
+    void baseVersion;
+
+    console.log(`✓ Published ${name}@${versionToPublish}`);
+    published.push({ name, version: versionToPublish });
+  }
+
+  return { published, skipped };
+}