@celilo/cli 3.1.0 → 4.1.0

package/src/services/static-content-converge.ts

@@ -1,390 +0,0 @@
-/**
- * The static-content converge (openspec/changes/capability-owned-tables,
- * stage 4 / design D10).
- *
- * `/srv/www` on the public_web provider stops being whatever the last
- * hand-built ssh upload left there and becomes convergent state: every
- * declared static route's release directory exists, `/srv/www/<slug>` points
- * at the current one through an atomic symlink swap, and everything past the
- * retention count is pruned. One Ansible converge makes the host match.
- *
- * Two callers, one code path:
- *  - the `public_web` capability's publish (`convergeStaticContent` callback
- *    injected by capability-loader), which runs the whole converge including
- *    `executeAnsible`, so a publish returns only once the host matches; and
- *  - the provider's OWN deploy (module-deploy), which writes the same release
- *    set into its generated inventory and lets the deploy's existing
- *    `executeAnsible` run the same role tasks — that is how a rebuilt host
- *    recovers with no consumer involvement (design D10, task 4.5).
- *
- * The desired state is the declared `web_routes` release set plus the source
- * directory core derived from `modules.source_path`. Nothing here writes into
- * a module tree: reading a module's web root is safe, writing is what breaks
- * `module verify` (celilo#1018, deliberately left to that thread).
- */
-
-import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';
-import { mkdir, writeFile } from 'node:fs/promises';
-import { join } from 'node:path';
-import { and, eq, isNotNull } from 'drizzle-orm';
-import { stringify as stringifyYaml } from 'yaml';
-import type { DbClient } from '../db/client';
-import { moduleConfigs, modules, webRoutes } from '../db/schema';
-import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
-import { parseStoredConfigValue } from './module-config';
-import { listPausedModules } from './module-pause';
-
-/** One content-hashed release the provider host must serve. */
-export interface StaticRelease {
-  /** The `/srv/www/<slug>` key. Independent of hostname on purpose. */
-  slug: string;
-  /** Content hash of the release — the release directory's suffix. */
-  contentHash: string;
-  /** Every FQDN served by this slug's routes (a slug spans hostnames). */
-  hostnames: string[];
-  /** Absolute celilo-mgr path core derived: `<module source_path>/site/dist`. */
-  sourceDir: string;
-  /**
-   * Absolute path to `<module source_path>/state/site` (celilo#1265), present
-   * ONLY when that directory exists. The role copies it over the release
-   * after the web root, so a hook-generated file wins over a built one. A
-   * module with no generated site content has no overlay and no row.
-   */
-  overlayDir?: string;
-}
-
-export interface StaticContentPlan {
-  releases: StaticRelease[];
-  /** How many release directories to keep per slug after a converge. */
-  retention: number;
-}
-
-/**
- * Retention when the operator has not set one. Mirrors the manifest default
- * declared on the provider module (`static_release_retention`); the manifest
- * is the operator-facing source of truth, this only covers a provider whose
- * manifest predates the variable.
- */
-const DEFAULT_STATIC_RELEASE_RETENTION = 5;
-
-export interface StaticContentConvergeResult {
-  success: boolean;
-  error?: string;
-  /** The written vars file, repo-absolute — for logs and tests. */
-  varsPath?: string;
-}
-
-/**
- * Where the converge's desired state lands inside the provider's generated
- * project. `group_vars/all/` is the existing auto-load directory (it already
- * holds `secrets.yml`), so no playbook change is needed to read it.
- */
-export function staticContentVarsPath(generatedPath: string): string {
-  return join(generatedPath, 'ansible', 'inventory', 'group_vars', 'all', 'static_content.yml');
-}
-
-/**
- * Resolve the retention count for the provider module.
- *
- * Operator config first (`static_release_retention` module config), then the
- * manifest's declared default, then the constant above. ONE resolution — the
- * role reads the variable bare and never carries a literal.
- */
-export function resolveStaticContentRetention(db: DbClient, providerModuleId: string): number {
-  const configRow = db
-    .select()
-    .from(moduleConfigs)
-    .where(
-      and(
-        eq(moduleConfigs.moduleId, providerModuleId),
-        eq(moduleConfigs.key, 'static_release_retention'),
-      ),
-    )
-    .get();
-  if (configRow) {
-    const parsed = parseStoredConfigValue(configRow);
-    const count = typeof parsed === 'number' ? parsed : Number.parseInt(String(parsed), 10);
-    if (Number.isInteger(count) && count >= 1) return count;
-    throw new Error(
-      `static_release_retention for ${providerModuleId} must be a positive integer, got: ${String(parsed)}`,
-    );
-  }
-
-  const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
-  if (module) {
-    const manifest = module.manifestData as {
-      variables?: { owns?: Array<{ name?: string; default?: unknown }> };
-    } | null;
-    const declared = manifest?.variables?.owns?.find((v) => v?.name === 'static_release_retention');
-    const fallback = typeof declared?.default === 'number' ? declared.default : undefined;
-    if (typeof fallback === 'number' && Number.isInteger(fallback) && fallback >= 1)
-      return fallback;
-  }
-
-  return DEFAULT_STATIC_RELEASE_RETENTION;
-}
-
-/**
- * Build the release set from the declared `web_routes` rows.
- *
- * Pure: reads the DB, decides nothing on the host (Rule 10.4). Grouping is by
- * slug, NOT by route — one slug is one release directory, and a slug can span
- * hostnames (lunacycle's two routes, both at `/`). A group that spans modules
- * or disagrees with itself about the hash is a contradiction the converge
- * cannot express, so it fails here with both rows named rather than silently
- * converging one of them.
- */
-export function planStaticContent(db: DbClient, publishingModuleId?: string): StaticRelease[] {
-  const rows = db
-    .select()
-    .from(webRoutes)
-    .where(and(eq(webRoutes.type, 'static'), isNotNull(webRoutes.contentHash)))
-    .all();
-
-  const bySlug = new Map<string, typeof rows>();
-  for (const row of rows) {
-    const group = bySlug.get(row.slug) ?? [];
-    group.push(row);
-    bySlug.set(row.slug, group);
-  }
-
-  // A paused module's site is left exactly as it is — pause preserves state
-  // (module-pause spec) — so its slug is neither converged nor validated. The
-  // exception is the module publishing right now: an unpause redeploy runs
-  // on_install while the module is still PAUSED (executeUnpause clears the
-  // pause only after the redeploy succeeds). celilo#1383.
-  const paused = new Set(listPausedModules(db).map((m) => m.id));
-  if (publishingModuleId) paused.delete(publishingModuleId);
-
-  const releases: StaticRelease[] = [];
-  for (const [slug, group] of bySlug) {
-    const moduleIds = [...new Set(group.map((r) => r.moduleId))];
-    if (moduleIds.length > 1) {
-      throw new Error(
-        `Static route slug "${slug}" is claimed by ${moduleIds.length} modules (${moduleIds.join(', ')}). The slug is the release key — /srv/www/${slug} cannot serve two release sets. Fix the colliding modules' registered paths.`,
-      );
-    }
-    const moduleId = moduleIds[0];
-    if (!moduleId) continue;
-    if (paused.has(moduleId)) continue;
-
-    const hashes = [...new Set(group.map((r) => r.contentHash))];
-    if (hashes.length > 1) {
-      throw new Error(
-        `Static route slug "${slug}" (module ${moduleId}) holds ${hashes.length} content hashes (${hashes.join(', ')}) — rows sharing a slug must share a release. Re-publish the module so every row carries the current hash.`,
-      );
-    }
-    const contentHash = hashes[0];
-    if (!contentHash) continue;
-
-    const sourceDir = resolveModuleWebRoot(moduleId, db);
-    if (!sourceDir) {
-      throw new Error(
-        `Static route slug "${slug}" belongs to module "${moduleId}", which is not installed — its release has no source to converge from.`,
-      );
-    }
-    if (!existsSync(sourceDir)) {
-      throw new Error(
-        `Static route slug "${slug}": no built site at ${sourceDir}. A module that serves a static site ships it at <module root>/site/dist. Redeploy the module.`,
-      );
-    }
-
-    // Present only when the module's hook actually wrote an overlay —
-    // otherwise the vars file would name a directory that does not exist
-    // and the role's copy task would have nothing to copy from.
-    const stateWebRoot = resolveModuleStateWebRoot(moduleId, db);
-    const overlayDir = stateWebRoot && existsSync(stateWebRoot) ? stateWebRoot : undefined;
-    releases.push({
-      slug,
-      contentHash,
-      hostnames: [...new Set(group.map((r) => r.hostname))].sort(),
-      sourceDir,
-      ...(overlayDir ? { overlayDir } : {}),
-    });
-  }
-
-  releases.sort((a, b) => a.slug.localeCompare(b.slug));
-  return releases;
-}
-
-/**
- * The full desired state: the release set plus the retention count resolved
- * for THIS provider module. The unit the vars file and the tests speak in.
- */
-export function buildStaticContentPlan(
-  db: DbClient,
-  providerModuleId: string,
-  publishingModuleId?: string,
-): StaticContentPlan {
-  return {
-    releases: planStaticContent(db, publishingModuleId),
-    retention: resolveStaticContentRetention(db, providerModuleId),
-  };
-}
-
-/**
- * Render the vars file. Presentation function (Rule 10.1) — deterministic
- * YAML, keyed by slug, carrying what 4.1 names: slug, content_hash, the
- * hostnames served, and the source directory core derived.
- */
-export function staticContentVarsYaml(plan: StaticContentPlan): string {
-  const vars = {
-    static_release_retention: plan.retention,
-    static_releases: plan.releases.map((r) => ({
-      slug: r.slug,
-      content_hash: r.contentHash,
-      hostnames: r.hostnames,
-      source_dir: r.sourceDir,
-      // Absent when there is no overlay, so the role's `when:` guard reads a
-      // clean variable — not an empty string a copy task would choke on.
-      ...(r.overlayDir ? { overlay_dir: r.overlayDir } : {}),
-    })),
-  };
-  const header =
-    '# Desired static-content state, generated by celilo (capability-owned-tables D10).\n' +
-    '# Hand edits are overwritten by the next converge. K = static_release_retention.\n';
-  return `${header}${stringifyYaml(vars, { lineWidth: 0, sortMapEntries: true })}`;
-}
-
-/**
- * Plan the release set and write it into the provider's generated inventory.
- *
- * The shared half of both converge triggers: the capability's callback adds
- * `executeAnsible` after this, and the provider's own deploy already runs it.
- * Nothing inspects verbatim role assets any more — the staleness pre-flight
- * retired with the ephemeral generated tree (D5 of
- * control-plane-stops-building-modules), so an injected vars file cannot read
- * as a stale generation because nothing reads staleness.
- */
-export async function writeStaticContentVars(
-  db: DbClient,
-  providerModuleId: string,
-  generatedPath: string,
-  publishingModuleId?: string,
-): Promise<StaticContentConvergeResult> {
-  const varsPath = staticContentVarsPath(generatedPath);
-
-  const plan = buildStaticContentPlan(db, providerModuleId, publishingModuleId);
-
-  await mkdir(join(varsPath, '..'), { recursive: true });
-  await writeFile(varsPath, staticContentVarsYaml(plan), 'utf-8');
-  return { success: true, varsPath };
-}
-
-/**
- * The full converge: plan, write the desired state, run Ansible against the
- * provider's generated project.
- *
- * `generatedPath` must already exist — it is the project the provider's last
- * deploy generated. If the project predates the converge role tasks, the
- * written state would be read by nothing and the transfer would silently
- * never happen, so the playbook is checked for the converge's own variable
- * first and a stale project fails loudly with the redeploy that fixes it.
- */
-/**
- * Whether any text file in a generated Ansible tree mentions `needle`.
- *
- * Scans the TREE rather than `playbook.yml`, because a provider's converge
- * belongs in one of its role's task files and a generated `playbook.yml` is a
- * thin hosts/vars/roles stanza that names no variable at all. caddy is the
- * worked example: `static_releases` appears in
- * `roles/caddy/tasks/static-content.yml`, `.../main.yml.tpl` and
- * `.../converge-release.yml`, and zero times in its playbook.
- *
- * @psbanka - 2026-09: this used to read `ansible/playbook.yml` alone, which
- * could not reach the string it was looking for. The guard below therefore
- * refused EVERY provider that had the converge support, and its remedy told the
- * operator to redeploy — which regenerates the same thin playbook and cannot
- * help. Guard and role tasks landed in the same commit (1fd3595e), so it had
- * never once passed, and it held three e2e suites red where nobody was looking.
- * celilo#1248.
- */
-export function ansibleTreeMentions(dir: string, needle: string): boolean {
-  if (!existsSync(dir)) return false;
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    const full = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      if (ansibleTreeMentions(full, needle)) return true;
-      continue;
-    }
-    // Text only. A generated tree can carry static assets, and reading those
-    // as utf-8 to look for a variable name is waste at best.
-    if (!/\.(ya?ml|j2|tpl|cfg|ini|conf)$/.test(entry.name)) continue;
-    if (readFileSync(full, 'utf-8').includes(needle)) return true;
-  }
-  return false;
-}
-
-export async function convergeStaticContent(
-  db: DbClient,
-  providerModuleId: string,
-  publishingModuleId?: string,
-): Promise<StaticContentConvergeResult> {
-  const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
-  if (!module) {
-    return { success: false, error: `Provider module '${providerModuleId}' is not installed` };
-  }
-  const generatedPath = join(module.sourcePath, 'generated');
-  // D4 (control-plane-stops-building-modules): the generated project is
-  // ephemeral, so a successful provider deploy deletes it and nothing is
-  // persisted for this converge to read. Render it here, on demand, from the
-  // signed payload plus resolved config — generation is fast and local. A
-  // render failure fails loudly with the deploy-once remedy, same as the old
-  // missing-project refusal did; a tree that predates the converge support is
-  // still refused by the static_releases check below.
-  if (!existsSync(generatedPath)) {
-    const { generateTemplates } = await import('../templates/generator');
-    const rendered = await generateTemplates({
-      moduleId: providerModuleId,
-      modulePath: module.sourcePath,
-      outputPath: generatedPath,
-      db,
-      skipVariableValidation: false,
-    });
-    if (!rendered.success) {
-      return {
-        success: false,
-        error:
-          `public_web provider '${providerModuleId}' has no generated deploy project at ${generatedPath} ` +
-          `and rendering one failed: ${rendered.error ?? 'unknown error'}. ` +
-          `Run \`celilo module deploy ${providerModuleId}\` once, then retry the publish.`,
-      };
-    }
-  }
-
-  const written = await writeStaticContentVars(
-    db,
-    providerModuleId,
-    generatedPath,
-    publishingModuleId,
-  );
-
-  // A generated project that predates the converge role tasks would read the
-  // new vars file and converge nothing. `static_releases` appearing nowhere in
-  // the project is the tell. Generic string check — core names no module.
-  const ansiblePath = join(generatedPath, 'ansible');
-  if (existsSync(ansiblePath) && !ansibleTreeMentions(ansiblePath, 'static_releases')) {
-    return {
-      success: false,
-      error: `The generated project for '${providerModuleId}' predates the static-content converge (its playbook never reads static_releases). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry the publish.`,
-    };
-  }
-
-  const { executeAnsible } = await import('./deploy-ansible');
-  // Tag-scoped: the full playbook re-templates the bootstrap Caddyfile, which
-  // would replace the real config the provider's reconcile wrote and reload
-  // caddy into serving nothing. A publish converge touches /srv/www only.
-  const result = await executeAnsible(generatedPath, { tags: ['static_content'] });
-  if (!result.success) {
-    return {
-      success: false,
-      error: `Static-content converge failed on ${providerModuleId}: ${result.error ?? 'unknown error'}`,
-      varsPath: written.varsPath,
-    };
-  }
-  // D4: the converge rendered the tree, used it, and is done with it. The same
-  // no-secrets-at-rest rule the deploy follows applies here; on the failure
-  // paths above the tree stays for inspection.
-  rmSync(generatedPath, { recursive: true, force: true });
-  return { success: true, varsPath: written.varsPath };
-}