@celilo/cli 0.3.30 → 0.4.0-alpha.0

package/src/services/aspect-runner.ts

@@ -0,0 +1,438 @@
+/**
+ * Aspect runner — executes a module's base-module aspect across
+ * the fleet.
+ *
+ * Per v2/CELILO_BASE.md Phase 1: when a module with an approved
+ * `base_module_aspect` triggers a fan-out event, the runner walks
+ * every non-`api_only` system in the aspect's `applicable_zones`,
+ * materializes an Ansible inventory + playbook in a scratch
+ * directory, copies the aspect role files there, and invokes
+ * `executeAnsible` against the result. The existing Ansible
+ * machinery does the actual SSH + playbook execution; the runner
+ * only sets up the per-aspect Ansible workspace.
+ *
+ * Scope NOT covered in SC3:
+ *  - Container_service (Proxmox LXC) systems are deferred to a
+ *    follow-up — `getSystemsByZone` covers the machines table only.
+ *  - Proxmox `nameserver` reconciliation is SC5.
+ *  - Trigger wiring (when `on_install` / `on_new_system_in_zone`
+ *    etc. fire) is SC4. SC3 ships the pure execution surface so
+ *    SC4 can call it from the deploy planner.
+ *  - Capability data injection into aspect host_vars is Phase 2+.
+ */
+
+import { existsSync } from 'node:fs';
+import { cp, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { type InventoryHost, generateHostsIni } from '../ansible/inventory';
+import { log } from '../cli/prompts';
+import type { getDb } from '../db/client';
+import { modules } from '../db/schema';
+import type { BaseModuleAspect, BaseModuleAspectTrigger, ModuleManifest } from '../manifest/schema';
+import type { Machine } from '../types/infrastructure';
+import { checkAspectApproval } from './aspect-approvals';
+import { resolveAspectTemplateRecord } from './aspect-template-resolver';
+import { executeAnsible } from './deploy-ansible';
+import { getSystemsByZone } from './machine-pool';
+import { executeProxmoxReconcile, planProxmoxReconcile } from './proxmox-reconcile';
+import { writeTemporarySshKey } from './ssh-key-manager';
+
+type DbClient = ReturnType<typeof getDb>;
+
+export interface AspectFanOutPlan {
+  /** Systems the aspect will run on. */
+  targetSystems: Machine[];
+  /** Systems that matched the zones but were excluded (api_only, etc.). */
+  skipped: Array<{ machine: Machine; reason: string }>;
+}
+
+export interface AspectRunResult {
+  success: boolean;
+  /** Ansible recap-level summary, or empty string if nothing ran. */
+  output: string;
+  /** Populated when success === false. */
+  error?: string;
+  /** The plan that was executed, for caller logging / event emission. */
+  plan: AspectFanOutPlan;
+}
+
+export interface AspectRunOptions {
+  /** Which trigger caused this fan-out. Recorded for logging. */
+  trigger: BaseModuleAspectTrigger;
+  /**
+   * Hostnames to exclude from fan-out, on top of `api_only`. The
+   * primary deploy's own host(s) typically don't need the aspect
+   * (per v2/CELILO_BASE.md D3 — aspect authors handle this when
+   * they want it, the framework doesn't force a skip).
+   */
+  excludeHostnames?: string[];
+  /**
+   * Override for noInteractive mode passed to executeAnsible.
+   * Defaults to `true` because aspect fan-outs run as part of
+   * larger orchestration (deploy flow) — there's no operator
+   * waiting at this specific step.
+   */
+  noInteractive?: boolean;
+}
+
+/**
+ * Planning-phase function (Rule 10.1): resolves which systems the
+ * aspect will target, without executing anything. Pure aside from
+ * the database read.
+ */
+export async function planAspectFanOut(
+  aspect: BaseModuleAspect,
+  options: Pick<AspectRunOptions, 'excludeHostnames'> = {},
+): Promise<AspectFanOutPlan> {
+  // Pull the candidate set with api_only already filtered out so
+  // we can record the skips separately for observability.
+  const allInZones = await getSystemsByZone(aspect.applicable_zones, {
+    excludeApiOnly: false,
+    excludeHostnames: options.excludeHostnames,
+  });
+
+  const targetSystems: Machine[] = [];
+  const skipped: AspectFanOutPlan['skipped'] = [];
+  for (const m of allInZones) {
+    if (m.apiOnly) {
+      skipped.push({ machine: m, reason: 'api_only' });
+    } else {
+      targetSystems.push(m);
+    }
+  }
+
+  return { targetSystems, skipped };
+}
+
+/**
+ * Materialize the per-aspect Ansible workspace in a temp dir:
+ *
+ *   <tmp>/
+ *     ansible/
+ *       inventory/
+ *         hosts.ini
+ *         host_vars/<hostname>.yml      (target_zone fact)
+ *         group_vars/all/aspect_vars.yml  (resolved ansible_vars
+ *                                          from the manifest, if any)
+ *       playbook.yml                    (synthesized)
+ *       roles/<aspect_role>/            (copied from the module's
+ *                                        base-module-aspect/ tree)
+ *
+ * Returns the temp dir path; caller is responsible for cleanup.
+ * Throws if the module's aspect role doesn't exist on disk.
+ *
+ * When `aspect.ansible_vars` is set, each template resolves against
+ * the providing module's context (its module_configs, capability
+ * data, system_config) and lands in group_vars/all/aspect_vars.yml
+ * — readable from the role as `{{ var_name }}`.
+ */
+export async function materializeAspectAnsible(args: {
+  aspect: BaseModuleAspect;
+  /** Absolute path to the module's imported source dir. The aspect
+   *  role is expected at `<moduleSourcePath>/base-module-aspect/ansible/roles/<aspect.ansible_role>`. */
+  moduleSourcePath: string;
+  targetSystems: Machine[];
+  /** Provider module ID — used to resolve $self / $capability /
+   *  $system references in ansible_vars. */
+  providerModuleId?: string;
+  /** DB client for context resolution. Required when ansible_vars
+   *  is declared on the aspect. */
+  db?: DbClient;
+}): Promise<string> {
+  const { aspect, moduleSourcePath, targetSystems, providerModuleId, db } = args;
+  const roleSrcDir = join(
+    moduleSourcePath,
+    'base-module-aspect',
+    'ansible',
+    'roles',
+    aspect.ansible_role,
+  );
+  if (!existsSync(roleSrcDir)) {
+    throw new Error(
+      `Aspect role not found at ${roleSrcDir}. The module declared base_module_aspect.ansible_role='${aspect.ansible_role}' but the corresponding directory is missing.`,
+    );
+  }
+
+  const workDir = await mkdtemp(join(tmpdir(), 'celilo-aspect-'));
+  const ansibleDir = join(workDir, 'ansible');
+  const inventoryDir = join(ansibleDir, 'inventory');
+  const hostVarsDir = join(inventoryDir, 'host_vars');
+  const rolesDir = join(ansibleDir, 'roles');
+  await mkdir(hostVarsDir, { recursive: true });
+  await mkdir(rolesDir, { recursive: true });
+
+  // Stage each system's SSH key and build inventory rows.
+  const inventoryHosts: InventoryHost[] = [];
+  for (const m of targetSystems) {
+    const keyPath = await writeTemporarySshKey(m.id);
+    inventoryHosts.push({
+      hostname: m.hostname,
+      ansibleHost: m.ipAddress,
+      ansibleUser: m.sshUser,
+      groups: ['aspect_targets', m.zone],
+      ansibleSshPrivateKeyFile: keyPath,
+    });
+
+    // Per-host vars: target_zone is the only fact the framework
+    // guarantees today (per D3). Capability-data / module-config
+    // injection is Phase 2+.
+    const hostVars = `---\n# Aspect host vars for ${m.hostname}\ntarget_zone: ${m.zone}\n`;
+    await writeFile(join(hostVarsDir, `${m.hostname}.yml`), hostVars, 'utf-8');
+  }
+
+  // hosts.ini groups every target under 'aspect_targets' and the
+  // per-system zone. The playbook (below) targets 'aspect_targets'.
+  const hostsIni = generateHostsIni(inventoryHosts);
+  await writeFile(join(inventoryDir, 'hosts.ini'), hostsIni, 'utf-8');
+
+  // Resolve and write aspect ansible_vars (if any) to
+  // group_vars/all/aspect_vars.yml. Every target reads these as
+  // `{{ var_name }}` from the role. Values resolve against the
+  // providing module's context — its module_configs, capability
+  // data, system_config — so the role sees concrete strings.
+  if (aspect.ansible_vars && Object.keys(aspect.ansible_vars).length > 0) {
+    if (!providerModuleId || !db) {
+      throw new Error(
+        'materializeAspectAnsible: aspect declares ansible_vars but providerModuleId/db were not supplied. This is a framework bug.',
+      );
+    }
+    const resolved = await resolveAspectTemplateRecord(
+      aspect.ansible_vars,
+      providerModuleId,
+      db,
+      'base_module_aspect.ansible_vars',
+    );
+    const groupVarsAllDir = join(inventoryDir, 'group_vars', 'all');
+    await mkdir(groupVarsAllDir, { recursive: true });
+    const lines = [
+      '---',
+      `# Resolved base_module_aspect.ansible_vars for ${providerModuleId}`,
+      ...Object.entries(resolved).map(([k, v]) => `${k}: ${JSON.stringify(v)}`),
+      '',
+    ];
+    await writeFile(join(groupVarsAllDir, 'aspect_vars.yml'), lines.join('\n'), 'utf-8');
+  }
+
+  // Copy the role tree into the staging dir so Ansible can find it
+  // via the default role path. cp -r equivalent; the role directory
+  // structure (tasks/, templates/, handlers/, vars/, defaults/)
+  // comes along intact.
+  await cp(roleSrcDir, join(rolesDir, aspect.ansible_role), { recursive: true });
+
+  // Synthesize the playbook. One play, targeting every host in the
+  // 'aspect_targets' group, invoking the single role. Become true
+  // because aspect roles typically modify /etc/* files.
+  const playbook = [
+    '---',
+    `- name: Aspect '${aspect.ansible_role}' fan-out`,
+    '  hosts: aspect_targets',
+    '  become: true',
+    '  gather_facts: true',
+    '  roles:',
+    `    - role: ${aspect.ansible_role}`,
+    '',
+  ].join('\n');
+  await writeFile(join(ansibleDir, 'playbook.yml'), playbook, 'utf-8');
+
+  return workDir;
+}
+
+/**
+ * Execution-phase function: orchestrates plan + materialize + run.
+ *
+ * Failure semantics (v2/CELILO_BASE.md D4): aspects are idempotent
+ * and forward-progress only. A failed fan-out is reported and the
+ * partial state (some systems updated, others not) is preserved —
+ * no rollback. The caller (deploy planner, in SC4) decides how to
+ * surface the failure.
+ */
+export async function runAspectFanOut(args: {
+  moduleId: string;
+  aspect: BaseModuleAspect;
+  moduleSourcePath: string;
+  options: AspectRunOptions;
+  db: DbClient;
+}): Promise<AspectRunResult> {
+  const { moduleId, aspect, moduleSourcePath, options, db } = args;
+
+  const plan = await planAspectFanOut(aspect, {
+    excludeHostnames: options.excludeHostnames,
+  });
+
+  if (plan.targetSystems.length === 0) {
+    log.info(
+      `Aspect fan-out for '${moduleId}' (${options.trigger}): no eligible systems in zones [${aspect.applicable_zones.join(', ')}]`,
+    );
+    return { success: true, output: '', plan };
+  }
+
+  log.info(
+    `Aspect fan-out for '${moduleId}' (${options.trigger}): ${plan.targetSystems.length} target(s) in zones [${aspect.applicable_zones.join(', ')}]`,
+  );
+
+  let workDir: string | undefined;
+  try {
+    workDir = await materializeAspectAnsible({
+      aspect,
+      moduleSourcePath,
+      targetSystems: plan.targetSystems,
+      providerModuleId: moduleId,
+      db,
+    });
+
+    const result = await executeAnsible(workDir, {
+      noInteractive: options.noInteractive ?? true,
+    });
+
+    // Proxmox reconciliation (D5): if the aspect declares
+    // proxmox_reconcile.tfvars and the fan-out plan includes
+    // Proxmox-provisioned LXCs, surface what the persisted
+    // terraform config WOULD need to look like. Currently
+    // observation-only (see proxmox-reconcile.ts header); when the
+    // persistence layer lands the planning here stays unchanged.
+    //
+    // Only attempt reconciliation if the Ansible run succeeded —
+    // there's no point warning about persisted-config drift if
+    // the running config didn't update.
+    if (result.success && aspect.proxmox_reconcile) {
+      try {
+        const reconcilePlan = await planProxmoxReconcile({
+          aspect,
+          providerModuleId: moduleId,
+          db,
+        });
+        executeProxmoxReconcile(reconcilePlan);
+      } catch (err) {
+        // Reconciliation planning failed (e.g., capability data
+        // missing). Don't fail the fan-out — the running config
+        // is already updated. Just warn.
+        log.warn(
+          `Proxmox reconciliation planning failed for '${moduleId}': ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    }
+
+    return {
+      success: result.success,
+      output: result.output,
+      error: result.error,
+      plan,
+    };
+  } finally {
+    if (workDir) {
+      try {
+        await rm(workDir, { recursive: true, force: true });
+      } catch {
+        // Best-effort cleanup. If rm fails the tmpdir GC will handle it.
+      }
+    }
+  }
+}
+
+/**
+ * Reasons the deploy planner might skip a fan-out without raising
+ * an error. Each is a "this is fine, just not applicable" outcome
+ * that the planner should log but not treat as a deploy failure.
+ */
+export type AspectSkipReason =
+  | 'no_aspect' // module didn't declare base_module_aspect
+  | 'trigger_not_declared' // aspect.triggers doesn't include this trigger
+  | 'no_approval' // operator hasn't approved (D2)
+  | 'scope_changed'; // approval exists but applicable_zones/triggers diverged (D7)
+
+export interface AspectGlueResult {
+  ran: boolean;
+  /** Populated when `ran === true`. */
+  success?: boolean;
+  /** Populated when `ran === false`. */
+  reason?: AspectSkipReason;
+  /** Populated when `ran === true` — the fan-out plan + Ansible recap. */
+  runResult?: AspectRunResult;
+}
+
+/**
+ * Deploy-flow glue (SC4): consulted by `module-deploy.ts` after a
+ * primary deploy successfully completes. Decides whether to fan an
+ * aspect out for the given trigger and dispatches if so.
+ *
+ * Gating logic, in order:
+ *
+ *   1. No `base_module_aspect` in the manifest → skip
+ *      (`reason: 'no_aspect'`).
+ *   2. The aspect's `triggers` list doesn't include the current
+ *      trigger → skip (`reason: 'trigger_not_declared'`).
+ *   3. No `aspect_approvals` row for (moduleId, version) → skip
+ *      (`reason: 'no_approval'`). Surface a warning so the operator
+ *      can re-import to grant consent.
+ *   4. Approval exists but the manifest's scope no longer matches
+ *      the approved scope_hash → skip (`reason: 'scope_changed'`).
+ *      Surface a warning that re-approval is required (D7).
+ *   5. Otherwise: invoke the runner with the named trigger.
+ *
+ * `runner` is injectable so unit tests don't need to drive real
+ * Ansible — the default is `runAspectFanOut`.
+ *
+ * Failure semantics (per D4): a failed fan-out is reported as
+ * `{ ran: true, success: false, ... }`. The PRIMARY deploy does
+ * not get rolled back — aspects are forward-progress only and a
+ * partial fleet update is expected to converge on the next
+ * fan-out. The caller (module-deploy.ts) should log the failure
+ * loudly but not change the primary deploy's success status.
+ */
+export async function maybeRunAspectForTrigger(args: {
+  moduleId: string;
+  manifest: ModuleManifest;
+  trigger: BaseModuleAspectTrigger;
+  db: DbClient;
+  runner?: typeof runAspectFanOut;
+  excludeHostnames?: string[];
+}): Promise<AspectGlueResult> {
+  const { moduleId, manifest, trigger, db } = args;
+  const aspect = manifest.base_module_aspect;
+
+  if (!aspect) {
+    return { ran: false, reason: 'no_aspect' };
+  }
+
+  if (!aspect.triggers.includes(trigger)) {
+    return { ran: false, reason: 'trigger_not_declared' };
+  }
+
+  const moduleRow = db.select().from(modules).where(eq(modules.id, moduleId)).get();
+  if (!moduleRow) {
+    // The deploy flow just acted on this module — its absence
+    // would be a deeper bug. Surface as "no_approval" with a log
+    // because there's nothing meaningful to fan out to.
+    log.warn(`Aspect glue: module '${moduleId}' not found in DB, skipping fan-out`);
+    return { ran: false, reason: 'no_approval' };
+  }
+
+  const approvalStatus = checkAspectApproval(moduleId, moduleRow.version, aspect, db);
+  if (approvalStatus === 'no_approval') {
+    log.warn(
+      `Aspect for '${moduleId}' is declared but not approved. Re-import the module to grant consent; aspect skipped.`,
+    );
+    return { ran: false, reason: 'no_approval' };
+  }
+  if (approvalStatus === 'scope_changed') {
+    log.warn(
+      `Aspect for '${moduleId}@${moduleRow.version}' has scope changes from the prior approval. Re-import to re-approve; aspect skipped.`,
+    );
+    return { ran: false, reason: 'scope_changed' };
+  }
+
+  const runner = args.runner ?? runAspectFanOut;
+  const runResult = await runner({
+    moduleId,
+    aspect,
+    moduleSourcePath: moduleRow.sourcePath,
+    options: {
+      trigger,
+      excludeHostnames: args.excludeHostnames,
+    },
+    db,
+  });
+  return { ran: true, success: runResult.success, runResult };
+}

package/src/services/aspect-template-resolver.test.ts

@@ -0,0 +1,101 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+import { type DbClient, createDbClient } from '../db/client';
+import { moduleConfigs, modules } from '../db/schema';
+import { resolveAspectTemplate, resolveAspectTemplateRecord } from './aspect-template-resolver';
+import { upsertDeployedSystem } from './deployed-systems';
+
+const TEST_DB_PATH = './test-aspect-resolver.db';
+
+/**
+ * Regression coverage for the `$infra:<name>.<field>` selector inside
+ * base_module_aspect ansible_vars. This path has its own hand-rolled
+ * resolver (separate from variables/resolver.ts); it previously handled
+ * only $self:/$system:/$capability:, so knot's migrated
+ * `knot_server_ip: $infra:main.ipv4_address` leaked into resolv.conf
+ * unresolved and broke DNS on every fleet machine. v2/MODULE_SYSTEMS_ADDRESSING.md.
+ */
+function seedModuleWithSystem(db: DbClient, moduleId: string, ip: string): void {
+  db.insert(modules)
+    .values({
+      id: moduleId,
+      name: moduleId,
+      version: '1.0.0',
+      manifestData: {},
+      sourcePath: `/tmp/${moduleId}`,
+    })
+    .run();
+  db.insert(moduleConfigs)
+    .values({ moduleId, key: 'hostname', value: moduleId, valueJson: JSON.stringify(moduleId) })
+    .run();
+  upsertDeployedSystem(db, moduleId, {
+    name: 'main',
+    hostname: `${moduleId}-host`,
+    ipv4Address: ip,
+    zone: 'internal',
+    infraType: 'machine',
+  });
+}
+
+describe('resolveAspectTemplate — $infra: selector', () => {
+  let db: DbClient;
+
+  beforeEach(() => {
+    db = createDbClient({ path: TEST_DB_PATH });
+    seedModuleWithSystem(db, 'knot-unbound-internal', '192.168.0.10');
+  });
+
+  afterEach(async () => {
+    db.$client.close();
+    for (const suffix of ['', '-shm', '-wal']) {
+      const p = `${TEST_DB_PATH}${suffix}`;
+      if (existsSync(p)) await rm(p);
+    }
+  });
+
+  test('resolves $infra:main.ipv4_address to the recorded system IP', async () => {
+    const resolved = await resolveAspectTemplate(
+      '$infra:main.ipv4_address',
+      'knot-unbound-internal',
+      db,
+      'base_module_aspect.ansible_vars',
+    );
+    expect(resolved).toBe('192.168.0.10');
+  });
+
+  test('resolves the braced form and embeds in surrounding text', async () => {
+    const resolved = await resolveAspectTemplate(
+      'nameserver ${infra:main.ipv4_address}',
+      'knot-unbound-internal',
+      db,
+      'base_module_aspect.ansible_vars',
+    );
+    expect(resolved).toBe('nameserver 192.168.0.10');
+  });
+
+  test('resolveAspectTemplateRecord resolves $infra: across keys', async () => {
+    const resolved = await resolveAspectTemplateRecord(
+      { knot_server_ip: '$infra:main.ipv4_address', knot_host: '$infra:main.hostname' },
+      'knot-unbound-internal',
+      db,
+      'base_module_aspect.ansible_vars',
+    );
+    expect(resolved).toEqual({
+      knot_server_ip: '192.168.0.10',
+      knot_host: 'knot-unbound-internal-host',
+    });
+  });
+
+  test('throws on an unknown system name', async () => {
+    expect(
+      resolveAspectTemplate('$infra:replica.ipv4_address', 'knot-unbound-internal', db, 'scope'),
+    ).rejects.toThrow(/Cannot resolve \$infra:replica\.ipv4_address/);
+  });
+
+  test('throws on an unknown field', async () => {
+    expect(
+      resolveAspectTemplate('$infra:main.bogus_field', 'knot-unbound-internal', db, 'scope'),
+    ).rejects.toThrow(/Cannot resolve \$infra:main\.bogus_field/);
+  });
+});

package/src/services/aspect-template-resolver.ts

@@ -0,0 +1,122 @@
+/**
+ * Shared template resolution for base-module aspect declarations.
+ *
+ * Both `proxmox_reconcile.tfvars` (SC5) and `ansible_vars` (SC6)
+ * accept string templates with the same substitution rules celilo
+ * uses for manifest variables — `$self:`, `$capability:`,
+ * `$system:`. Resolution happens against the PROVIDING module's
+ * context: the values come from the module that owns the aspect,
+ * not the target system the aspect runs on.
+ *
+ * Factored out so the two callers stay consistent — a fix to the
+ * substitution rules lands here once.
+ */
+
+import type { getDb } from '../db/client';
+import { buildResolutionContext } from '../variables/context';
+
+type DbClient = ReturnType<typeof getDb>;
+
+/**
+ * Resolve a template string against `providerModuleId`'s context.
+ *
+ * `scopeLabel` is included in error messages so failures point at
+ * the right manifest block (e.g., 'proxmox_reconcile.tfvars',
+ * 'base_module_aspect.ansible_vars').
+ *
+ * Throws when a substitution references a value that doesn't
+ * exist — the caller is expected to surface this to the operator
+ * since it's an aspect-author bug.
+ */
+export async function resolveAspectTemplate(
+  template: string,
+  providerModuleId: string,
+  db: DbClient,
+  scopeLabel: string,
+): Promise<string> {
+  const ctx = await buildResolutionContext(providerModuleId, db);
+
+  let result = template;
+
+  result = result.replace(/\$\{?system:([a-zA-Z0-9_.]+)\}?/g, (_match, key) => {
+    const value = ctx.systemConfig[key];
+    if (value === undefined) {
+      throw new Error(
+        `Cannot resolve $system:${key} in ${scopeLabel} (provider module: ${providerModuleId})`,
+      );
+    }
+    return value;
+  });
+
+  result = result.replace(/\$\{?self:([a-zA-Z0-9_]+)\}?/g, (_match, key) => {
+    const value = ctx.selfConfig[key];
+    if (value === undefined) {
+      throw new Error(
+        `Cannot resolve $self:${key} in ${scopeLabel} (provider module: ${providerModuleId})`,
+      );
+    }
+    return value;
+  });
+
+  // $infra:<system-name>.<field> — the provider's deployed system, by name
+  // (v2/MODULE_SYSTEMS_ADDRESSING.md). This is how an aspect references the
+  // provider's own host IP (e.g. knot's dns-client-config aspect sets every
+  // fleet machine's nameserver to $infra:main.ipv4_address). Without this the
+  // literal template string would leak into resolv.conf.
+  result = result.replace(
+    /\$\{?infra:([a-z0-9-]+)\.([a-zA-Z0-9_]+)\}?/g,
+    (_match, sysName, field) => {
+      const sys = ctx.systems?.[sysName] as unknown as Record<string, unknown> | undefined;
+      const value = sys?.[field];
+      if (value === undefined || value === '') {
+        throw new Error(
+          `Cannot resolve $infra:${sysName}.${field} in ${scopeLabel} (provider module: ${providerModuleId}) — no such deployed system or field`,
+        );
+      }
+      return String(value);
+    },
+  );
+
+  result = result.replace(
+    /\$capability:([a-zA-Z0-9_]+)\.([a-zA-Z0-9_.]+)/g,
+    (_match, capName, path) => {
+      const capData = ctx.capabilities[capName];
+      if (!capData) {
+        throw new Error(
+          `Cannot resolve $capability:${capName}.${path} — capability not registered (provider module: ${providerModuleId}, in ${scopeLabel})`,
+        );
+      }
+      const parts = path.split('.');
+      let cur: unknown = capData;
+      for (const p of parts) {
+        if (cur && typeof cur === 'object' && p in (cur as Record<string, unknown>)) {
+          cur = (cur as Record<string, unknown>)[p];
+        } else {
+          throw new Error(
+            `Cannot resolve $capability:${capName}.${path} — field missing on capability data (provider module: ${providerModuleId}, in ${scopeLabel})`,
+          );
+        }
+      }
+      return String(cur);
+    },
+  );
+
+  return result;
+}
+
+/**
+ * Convenience: resolve a record-of-templates, returning a record
+ * of resolved strings. The keys pass through unchanged.
+ */
+export async function resolveAspectTemplateRecord(
+  templates: Record<string, string>,
+  providerModuleId: string,
+  db: DbClient,
+  scopeLabel: string,
+): Promise<Record<string, string>> {
+  const resolved: Record<string, string> = {};
+  for (const [name, template] of Object.entries(templates)) {
+    resolved[name] = await resolveAspectTemplate(template, providerModuleId, db, scopeLabel);
+  }
+  return resolved;
+}