@celilo/cli 0.9.1 → 0.10.0

package/CELILO_CORE_MODULES.md

@@ -44,7 +44,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
 - **forgejo** — self-hosted Forgejo git forge (git-over-SSH, OIDC, public ingress). **provides:** `source_forge`. **requires:** `public_web`, `idp`, `firewall`, `dns_registrar`. See `v2/CI_PIPELINE.md`.
 - **forgejo-runner** — host-mode (LXC) Forgejo Actions runner for unit/lint jobs; outbound-only, lives in dmz. **requires:** `source_forge`. See `v2/FORGEJO_RUNNER.md`.
 - **forgejo-builder** — VM-based, Docker-capable Forgejo Actions runner (`requires.system.type: vm`) for hermetic release builds + the cele2e suite. **requires:** `source_forge`.
-- **npm-cache-node** — self-hosted npm registry as a pass-through disk cache (proxies upstream). **requires:** `public_web`, `dns_registrar`. See `v2/NPM_CACHE_NODE.md`.
+- **npm-cache-node** — self-hosted npm registry: a pass-through disk cache (proxies upstream) plus a `PUT /<pkg>` publish endpoint that stores locally published `@celilo/*` tarballs as authoritative local-origin (never overwritten by upstream). PUT is gated on a static operator-set publish token (`publish_tokens` secret, SHA-256 model; fail-closed — absent/bad token → 401); reads stay open (it's a mirror). **requires:** `public_web`, `dns_registrar`. See `v2/NPM_CACHE_NODE.md`.
 
 ## Applications
 

package/CELILO_SUBSYSTEMS.md

@@ -81,3 +81,4 @@ see `design/README.md`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MO
 
 - **cele2e harness** — `packages/e2e/src/` — `runner.ts`, `container-manager.ts` (`startNetwork`, `reconnectNetwork`), `network-builder.ts` (`NetworkBuilder`).
 - **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
+- **MCP server (agent-driven e2e)** — `packages/mcp-server/src/index.ts` — stdio MCP server exposing `start_run`/`run_status`/`run_result`/`stop_run` (detached cele2e runs read off the event bus, no ANSI scraping) + `env_check` (docker VM, run-lock, shared-infra, mgmt image CLI version, netapps). Dev/ops tool, `private`, not shipped to consumers.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -16,7 +16,13 @@
     "CELILO_SUBSYSTEMS.md",
     "CELILO_CORE_MODULES.md"
   ],
-  "keywords": ["celilo", "homelab", "orchestration", "ansible", "terraform"],
+  "keywords": [
+    "celilo",
+    "homelab",
+    "orchestration",
+    "ansible",
+    "terraform"
+  ],
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -51,7 +57,7 @@
     "@aws-sdk/client-s3": "^3.1024.0",
     "@celilo/capabilities": "^0.6.0",
     "@celilo/cli-display": "^0.1.9",
-    "@celilo/event-bus": "^0.1.7",
+    "@celilo/event-bus": "^0.1.8",
     "@clack/prompts": "^1.1.0",
     "ajv": "^8.18.0",
     "drizzle-orm": "^0.36.4",

package/src/cli/commands/publish/helpers.ts

@@ -125,6 +125,24 @@ export function readExternalProjectPaths(): string[] {
   return [];
 }
 
+/**
+ * Read `NPM_PUBLISH_TARGET` from .env — the registry URL `celilo publish`
+ * points `bun publish --registry` at for @celilo/* packages (a deployed
+ * npm-cache-node). Unset → publish to npmjs (current behavior). See
+ * v2/NPM_CACHE_NODE.md Phase 3.1 / v2/PUBLILO_CLI.md decision 10.
+ */
+export function readNpmPublishTarget(): string | null {
+  if (!existsSync(ENV_FILE)) return null;
+  const content = readFileSync(ENV_FILE, 'utf-8');
+  for (const line of content.split('\n')) {
+    const m = line.match(/^\s*NPM_PUBLISH_TARGET\s*=\s*(.+?)\s*$/);
+    if (!m) continue;
+    const raw = m[1].replace(/^["']|["']$/g, '').trim();
+    return raw || null;
+  }
+  return null;
+}
+
 /**
  * Recursively find every package.json under a root, skipping
  * node_modules and common build-output dirs (so we don't try to rewrite

package/src/cli/commands/publish/types.ts

@@ -139,15 +139,13 @@ export interface RewriteOptions {
 }
 
 /**
- * Per-package pre-publish work. Currently only `@celilo/e2e` triggers
- * any of these (it ships netapps + registry server + caches inside its
- * tarball), but the plan still enumerates them so dry-run can show
- * what extra work the publish entails.
+ * Per-package pre-publish work. Currently only `@celilo/e2e` triggers this
+ * (it bundles the npm-compat registry server source inside its tarball).
+ * Sim-content caches (website/npm) and standard-module netapps are NOT
+ * bundled at publish — they're fetched from the public celilo sources at
+ * `cele2e build-infra` time instead (ce-qwz Decisions 2B + 3, ce-i2i).
  */
-export type PrePublishHookKind =
-  | 'registryServerBundle'
-  | 'rebuildE2eNetapps'
-  | 'stageE2ePublishCaches';
+export type PrePublishHookKind = 'registryServerBundle';
 
 /**
  * Single workspace package planned to publish (or explicitly skip) in

package/src/cli/commands/publish/workspace.test.ts

@@ -57,6 +57,7 @@ mock.module('./helpers', () => ({
   },
   listModuleDirs: () => [],
   readExternalProjectPaths: () => [],
+  readNpmPublishTarget: () => null,
   findPackageJsons: () => [],
   bareVersion: (s: string) => s.replace(/^[\s^~=><]+/, '').trim(),
   withOperator: (oldSpec: string, newVersion: string) => {
@@ -74,6 +75,10 @@ mock.module('./alpha', () => ({
     return { name: spec.slice(0, i), version: spec.slice(i + 1) };
   },
   stripAlphaSuffix: (v: string) => v.replace(/-alpha\.\d+$/, ''),
+  prereleaseDistTag: (v: string) => {
+    const dash = v.indexOf('-');
+    return dash === -1 ? undefined : v.slice(dash + 1).split('.')[0] || undefined;
+  },
   isAlphaVersion: (v: string) => /-alpha\.\d+$/.test(v),
   nextAlphaNumber: (name: string, semverCore: string) =>
     nextAlphaResponses[`${name}@${semverCore}`] ?? 0,
@@ -84,7 +89,7 @@ mock.module('./alpha', () => ({
   decideAlphaSkip: () => ({ skip: false }),
 }));
 
-const { planWorkspace } = await import('./workspace');
+const { planWorkspace, buildPublishArgs } = await import('./workspace');
 
 function buildBaseMap(): Map<string, string> {
   const m = new Map<string, string>();
@@ -131,7 +136,7 @@ describe('planWorkspace', () => {
       expect(cli?.skipReason).toBeUndefined();
     });
 
-    test('e2e gets the registry-bundle + netapps + cache hooks', () => {
+    test('e2e gets only the registry-bundle hook (caches/netapps fetched at build-infra, not bundled)', () => {
       publishedSet = new Set();
       const result = planWorkspace({
         mode: { kind: 'normal' },
@@ -141,11 +146,7 @@ describe('planWorkspace', () => {
       });
 
       const e2e = result.items.find((i) => i.pkg === 'packages/e2e');
-      expect(e2e?.hooks).toEqual([
-        'registryServerBundle',
-        'rebuildE2eNetapps',
-        'stageE2ePublishCaches',
-      ]);
+      expect(e2e?.hooks).toEqual(['registryServerBundle']);
       const cli = result.items.find((i) => i.pkg === 'apps/celilo');
       expect(cli?.hooks).toEqual([]);
     });
@@ -302,6 +303,42 @@ describe('planWorkspace', () => {
     });
   });
 
+  describe('buildPublishArgs registry target', () => {
+    test('no target → default registry (unchanged behavior)', () => {
+      expect(buildPublishArgs({ name: '@celilo/cli', tag: undefined }, null)).toEqual([
+        'publish',
+        '--access',
+        'public',
+      ]);
+    });
+
+    test('target set → --registry appended for @celilo/* package', () => {
+      expect(
+        buildPublishArgs({ name: '@celilo/cli', tag: undefined }, 'https://npm.example.test/'),
+      ).toEqual(['publish', '--access', 'public', '--registry', 'https://npm.example.test/']);
+    });
+
+    test('tag and registry both present', () => {
+      expect(
+        buildPublishArgs({ name: '@celilo/cli', tag: 'alpha' }, 'https://npm.example.test/'),
+      ).toEqual([
+        'publish',
+        '--access',
+        'public',
+        '--tag',
+        'alpha',
+        '--registry',
+        'https://npm.example.test/',
+      ]);
+    });
+
+    test('non-@celilo package never gets the private registry', () => {
+      expect(
+        buildPublishArgs({ name: 'some-other-pkg', tag: undefined }, 'https://npm.example.test/'),
+      ).toEqual(['publish', '--access', 'public']);
+    });
+  });
+
   describe('dependency order', () => {
     test('preserves PACKAGES order in planned items', () => {
       publishedSet = new Set();

package/src/cli/commands/publish/workspace.ts

@@ -17,18 +17,8 @@
  */
 
 import { spawnSync } from 'node:child_process';
-import {
-  cpSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  readdirSync,
-  rmSync,
-  statSync,
-  unlinkSync,
-  writeFileSync,
-} from 'node:fs';
-import { join, relative } from 'node:path';
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
 import {
   ALPHA_TAG,
   alphaSkipDecision,
@@ -36,7 +26,7 @@ import {
   prereleaseDistTag,
   stripAlphaSuffix,
 } from './alpha';
-import { REPO_ROOT, isPublished, readPkg } from './helpers';
+import { REPO_ROOT, isPublished, readNpmPublishTarget, readPkg } from './helpers';
 import type {
   PackageJson,
   PublishMode,
@@ -74,13 +64,15 @@ export interface PlanWorkspaceOutput {
 
 /**
  * Pre-publish hooks that fire for @celilo/e2e (the only package that
- * needs them today). Listed in the WorkspaceItem so dry-run can show
- * "@celilo/e2e (will rebuild netapps, stage caches)" without executing
- * anything.
+ * needs one today): refresh the bundled npm-compat registry-server source.
+ * Sim-content caches and standard-module netapps are no longer staged into
+ * the tarball at publish — a monorepo-free consumer fetches them from the
+ * public celilo sources at `cele2e build-infra` time (ce-qwz Decisions
+ * 2B + 3, ce-i2i).
  */
 function workspaceHooksFor(pkg: string): WorkspaceItem['hooks'] {
   if (pkg !== 'packages/e2e') return [];
-  return ['registryServerBundle', 'rebuildE2eNetapps', 'stageE2ePublishCaches'];
+  return ['registryServerBundle'];
 }
 
 /**
@@ -328,147 +320,23 @@ export async function verifyPublishedDeps(
   }
 }
 
-// ─── Per-package pre-publish hooks ─────────────────────────────────
-
-/**
- * Modules excluded from the @celilo/e2e netapps shipment:
- *   - celilo-registry: bundles the bun-compiled registry server
- *     binaries (~76 MB packed). Not used by typical consumer e2e
- *     tests; including it would bloat the npm tarball past the SSL
- *     transport's reliable window.
- *   - archive: not a real module dir.
- */
-const E2E_NETAPP_EXCLUDES = new Set(['archive', 'celilo-registry']);
-
-/**
- * Pre-publish step for @celilo/e2e: package each (non-excluded)
- * module under `<root>/modules/` into `packages/e2e/netapps/`. Replaces
- * whatever was last left there by a local `cele2e build-infra` so the
- * shipped tarball is always built from the current branch, not the
- * publisher's last local development build.
- */
-export function rebuildE2eNetapps(repoRoot: string): void {
-  const modulesDir = join(repoRoot, 'modules');
-  const netappsDir = join(repoRoot, 'packages/e2e/netapps');
-  const celiloWrapper = join(repoRoot, 'celilo');
-
-  if (!existsSync(modulesDir)) {
-    console.warn(
-      `⚠ ${modulesDir} not found — skipping netapp rebuild. The published tarball will reuse whatever's currently in packages/e2e/netapps/.`,
-    );
-    return;
-  }
-  if (!existsSync(celiloWrapper)) {
-    console.warn(
-      `⚠ ${celiloWrapper} not found — skipping netapp rebuild. Same staleness risk as above.`,
-    );
-    return;
-  }
-
-  console.log('Rebuilding packages/e2e/netapps/ from infra/modules/...');
-
-  for (const existing of readdirSync(netappsDir).filter((f) => f.endsWith('.netapp'))) {
-    try {
-      unlinkSync(join(netappsDir, existing));
-    } catch {
-      // Best effort
-    }
-  }
-
-  const moduleDirs = readdirSync(modulesDir).filter((name) => {
-    if (E2E_NETAPP_EXCLUDES.has(name)) return false;
-    const dir = join(modulesDir, name);
-    try {
-      return statSync(dir).isDirectory() && existsSync(join(dir, 'manifest.yml'));
-    } catch {
-      return false;
-    }
-  });
-
-  let okCount = 0;
-  const failures: Array<{ name: string; stderr: string }> = [];
-  for (const name of moduleDirs) {
-    const moduleDir = join(modulesDir, name);
-    const out = join(netappsDir, `${name}.netapp`);
-    const r = spawnSync(celiloWrapper, ['package', moduleDir, '--output', out], {
-      cwd: repoRoot,
-      stdio: ['ignore', 'pipe', 'pipe'],
-      encoding: 'utf-8',
-    });
-    if (r.status === 0) {
-      okCount++;
-    } else {
-      failures.push({ name, stderr: (r.stderr ?? '').trim() || `exit ${r.status}` });
-    }
-  }
-
-  if (failures.length > 0) {
-    console.error(`✗ Failed to package ${failures.length} module(s):`);
-    for (const f of failures) {
-      console.error(`  ${f.name}: ${f.stderr}`);
-    }
-    console.error(
-      'Aborting publish — shipping with missing netapps would silently break consumer e2e tests.',
-    );
-    process.exit(1);
-  }
-
-  console.log(
-    `Refreshed ${okCount} netapp(s) in packages/e2e/netapps/ (excluded: ${[...E2E_NETAPP_EXCLUDES].join(', ')}).\n`,
-  );
-}
-
 /**
- * Pre-publish step for @celilo/e2e: stage `.celilo-website-cache/` and
- * `.npm-registry-cache/` inside packages/e2e/ so the tarball ships with
- * everything Dockerfile.celilo-website-sim and Dockerfile.npm-registry-sim
- * need to COPY at docker-build time.
+ * Build the `bun publish` argv for a workspace item. When a private
+ * registry target is configured (a deployed npm-cache-node) AND the
+ * package is @celilo/*-scoped, point bun at it via --registry; otherwise
+ * publish to the default registry (npmjs — current behavior).
+ * v2/NPM_CACHE_NODE.md Phase 3.1 / v2/PUBLILO_CLI.md decision 10.
  */
-export function stageE2ePublishCaches(repoRoot: string): void {
-  const websiteSrc = join(repoRoot, 'modules', 'celilo-website', 'site');
-  const e2eDir = join(repoRoot, 'packages', 'e2e');
-  const websiteCache = join(e2eDir, '.celilo-website-cache');
-  const npmCache = join(e2eDir, '.npm-registry-cache');
-  const packScript = join(e2eDir, 'scripts', 'pack-celilo-packages.ts');
-
-  if (!existsSync(websiteSrc)) {
-    console.error(`✗ ${websiteSrc} not found — cannot stage .celilo-website-cache.`);
-    process.exit(1);
+export function buildPublishArgs(
+  item: Pick<WorkspaceItem, 'name' | 'tag'>,
+  registryTarget: string | null,
+): string[] {
+  const args = ['publish', '--access', 'public'];
+  if (item.tag) args.push('--tag', item.tag);
+  if (registryTarget && item.name.startsWith('@celilo/')) {
+    args.push('--registry', registryTarget);
   }
-  if (!existsSync(packScript)) {
-    console.error(`✗ ${packScript} not found — cannot stage .npm-registry-cache.`);
-    process.exit(1);
-  }
-
-  console.log('Staging .celilo-website-cache/ from modules/celilo-website/site/...');
-  const installResult = spawnSync('bun', ['install'], { cwd: websiteSrc, stdio: 'pipe' });
-  if (installResult.status !== 0) {
-    console.error('✗ bun install for celilo-website failed:');
-    console.error(installResult.stderr?.toString());
-    process.exit(1);
-  }
-  const buildResult = spawnSync('bun', ['run', 'build'], { cwd: websiteSrc, stdio: 'pipe' });
-  if (buildResult.status !== 0) {
-    console.error('✗ bun run build for celilo-website failed:');
-    console.error(buildResult.stderr?.toString());
-    process.exit(1);
-  }
-  rmSync(websiteCache, { recursive: true, force: true });
-  mkdirSync(websiteCache, { recursive: true });
-  cpSync(join(websiteSrc, 'dist'), websiteCache, { recursive: true });
-  console.log(`✓ Staged .celilo-website-cache/ (from ${relative(repoRoot, websiteSrc)}/dist/)\n`);
-
-  console.log('Staging .npm-registry-cache/ from @celilo/* workspace tarballs...');
-  const packResult = spawnSync('bun', ['run', packScript], { cwd: repoRoot, stdio: 'pipe' });
-  if (packResult.status !== 0) {
-    console.error('✗ pack-celilo-packages.ts failed:');
-    console.error(packResult.stderr?.toString());
-    process.exit(1);
-  }
-  const packed = existsSync(npmCache)
-    ? readdirSync(npmCache).filter((f) => f.endsWith('.tgz'))
-    : [];
-  console.log(`✓ Staged .npm-registry-cache/ with ${packed.length} workspace tarball(s)\n`);
+  return args;
 }
 
 // ─── Executor ──────────────────────────────────────────────────────
@@ -486,6 +354,13 @@ export async function executeWorkspace(input: ExecuteWorkspaceInput): Promise<Pu
   const published: PublishResult['published'] = [];
   const skipped: string[] = [];
 
+  // When an npm-cache-node is configured, @celilo/* tarballs go there
+  // (the cache forwards upstream under policy); unset → npmjs.
+  const registryTarget = readNpmPublishTarget();
+  if (registryTarget) {
+    console.log(`\nPublishing @celilo/* to configured registry: ${registryTarget}`);
+  }
+
   for (const item of items) {
     const { pkg, name, baseVersion, versionToPublish } = item;
 
@@ -511,12 +386,6 @@ export async function executeWorkspace(input: ExecuteWorkspaceInput): Promise<Pu
         'Refreshed packages/e2e/registry-server/ bundle from packages/registry-server.\n',
       );
     }
-    if (item.hooks.includes('rebuildE2eNetapps')) {
-      rebuildE2eNetapps(REPO_ROOT);
-    }
-    if (item.hooks.includes('stageE2ePublishCaches')) {
-      stageE2ePublishCaches(REPO_ROOT);
-    }
 
     const { original: pkgJsonOriginal, rewrites: workspaceRewrites } = rewriteWorkspaceDeps(
       pkg,
@@ -548,8 +417,7 @@ export async function executeWorkspace(input: ExecuteWorkspaceInput): Promise<Pu
       process.exit(1);
     }
 
-    const publishArgs = ['publish', '--access', 'public'];
-    if (item.tag) publishArgs.push('--tag', item.tag);
+    const publishArgs = buildPublishArgs(item, registryTarget);
 
     let publishStatus: number | null = null;
     let publishError: unknown = null;
@@ -591,7 +459,15 @@ export async function executeWorkspace(input: ExecuteWorkspaceInput): Promise<Pu
     }
 
     if (workspaceRewrites.length > 0) {
-      await verifyPublishedDeps(name, versionToPublish, workspaceRewrites);
+      // Verify against the same registry we published to — a cache node
+      // forwards upstream under policy, so the version may not be on
+      // npmjs yet.
+      await verifyPublishedDeps(
+        name,
+        versionToPublish,
+        workspaceRewrites,
+        registryTarget ?? undefined,
+      );
     }
 
     // Silence the unused-var warning on baseVersion — we keep the value