@celilo/cli 0.3.30 → 0.4.0-alpha.1

package/src/variables/computed/parse.ts

@@ -0,0 +1,262 @@
+/**
+ * Parser for the computed-variable DSL.
+ *
+ * Computed variables (v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md, D1) are
+ * declared in a manifest with `type: computed` and a `value:` expression
+ * derived on access from other values, e.g.:
+ *
+ *   value: "keys(secret.ddns_passwords)"
+ *   value: "unique(concat(self.a, self.b))"
+ *   value: "format('{host}.{zone}', host=self.hostname, zone=system.primary_domain)"
+ *
+ * This module turns that string into an AST. It is PURE — no evaluation,
+ * no lookups, no I/O. The grammar is deliberately tiny: a call over an
+ * allow-list of functions whose arguments are variable references, string
+ * literals, bare identifiers (field names), nested calls, or named args
+ * (for `format`). It is NOT a general expression language — there is no
+ * arithmetic, no control flow, no operators.
+ */
+
+/** Roots a variable reference may start with (`secret.ddns_passwords`). */
+export const REF_ROOTS = ['self', 'system', 'secret', 'system_secret', 'capability'] as const;
+export type RefRoot = (typeof REF_ROOTS)[number];
+
+const REF_ROOT_SET = new Set<string>(REF_ROOTS);
+
+/** A function call: `keys(secret.ddns_passwords)`. */
+export interface CallNode {
+  kind: 'call';
+  fn: string;
+  args: Arg[];
+}
+
+/** A variable reference whose first segment is a known root. */
+export interface RefNode {
+  kind: 'ref';
+  root: RefRoot;
+  /** Path segments after the root, e.g. `["ddns_passwords"]`. */
+  path: string[];
+}
+
+/** A quoted string literal: `'{host}.{zone}'`. */
+export interface StrNode {
+  kind: 'str';
+  value: string;
+}
+
+/**
+ * A bare (non-root) identifier, possibly dotted — used as a field-name
+ * argument, e.g. the `ip` in `map(self.upstreams, ip)`.
+ */
+export interface BareNode {
+  kind: 'bare';
+  name: string;
+}
+
+export type Node = CallNode | RefNode | StrNode | BareNode;
+
+/** A call argument — optionally named (`host=self.hostname`). */
+export interface Arg {
+  /** Present only for named args (used by `format`). */
+  name?: string;
+  value: Node;
+}
+
+export class ComputedParseError extends Error {
+  constructor(
+    message: string,
+    readonly position: number,
+  ) {
+    super(`Computed expression parse error at ${position}: ${message}`);
+    this.name = 'ComputedParseError';
+  }
+}
+
+type TokenType = 'ident' | 'string' | '(' | ')' | ',' | '=' | '.';
+
+interface Token {
+  type: TokenType;
+  value: string;
+  pos: number;
+}
+
+const IDENT_START = /[a-zA-Z_]/;
+const IDENT_CHAR = /[a-zA-Z0-9_]/;
+
+function tokenize(input: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+
+  while (i < input.length) {
+    const ch = input[i];
+
+    if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
+      i++;
+      continue;
+    }
+
+    if (ch === '(' || ch === ')' || ch === ',' || ch === '=' || ch === '.') {
+      tokens.push({ type: ch, value: ch, pos: i });
+      i++;
+      continue;
+    }
+
+    if (ch === "'" || ch === '"') {
+      const quote = ch;
+      const start = i;
+      i++;
+      let value = '';
+      while (i < input.length && input[i] !== quote) {
+        value += input[i];
+        i++;
+      }
+      if (i >= input.length) {
+        throw new ComputedParseError('unterminated string literal', start);
+      }
+      i++; // closing quote
+      tokens.push({ type: 'string', value, pos: start });
+      continue;
+    }
+
+    if (IDENT_START.test(ch)) {
+      const start = i;
+      let value = '';
+      while (i < input.length && IDENT_CHAR.test(input[i])) {
+        value += input[i];
+        i++;
+      }
+      tokens.push({ type: 'ident', value, pos: start });
+      continue;
+    }
+
+    throw new ComputedParseError(`unexpected character '${ch}'`, i);
+  }
+
+  return tokens;
+}
+
+class Parser {
+  private pos = 0;
+
+  constructor(private readonly tokens: Token[]) {}
+
+  parse(): Node {
+    const node = this.parseExpr();
+    if (this.pos < this.tokens.length) {
+      throw new ComputedParseError(
+        `unexpected trailing input '${this.tokens[this.pos].value}'`,
+        this.tokens[this.pos].pos,
+      );
+    }
+    return node;
+  }
+
+  private peek(): Token | undefined {
+    return this.tokens[this.pos];
+  }
+
+  private next(): Token {
+    const t = this.tokens[this.pos];
+    if (!t) {
+      throw new ComputedParseError('unexpected end of expression', -1);
+    }
+    this.pos++;
+    return t;
+  }
+
+  private expect(type: TokenType): Token {
+    const t = this.next();
+    if (t.type !== type) {
+      throw new ComputedParseError(`expected '${type}' but found '${t.value}'`, t.pos);
+    }
+    return t;
+  }
+
+  private parseExpr(): Node {
+    const t = this.peek();
+    if (!t) {
+      throw new ComputedParseError('expected an expression', -1);
+    }
+
+    if (t.type === 'string') {
+      this.next();
+      return { kind: 'str', value: t.value };
+    }
+
+    if (t.type === 'ident') {
+      // Function call?
+      if (this.tokens[this.pos + 1]?.type === '(') {
+        return this.parseCall();
+      }
+      return this.parseRefOrBare();
+    }
+
+    throw new ComputedParseError(`expected an expression but found '${t.value}'`, t.pos);
+  }
+
+  private parseCall(): CallNode {
+    const fnTok = this.expect('ident');
+    this.expect('(');
+    const args: Arg[] = [];
+
+    if (this.peek()?.type !== ')') {
+      for (;;) {
+        args.push(this.parseArg());
+        if (this.peek()?.type === ',') {
+          this.next();
+          continue;
+        }
+        break;
+      }
+    }
+
+    this.expect(')');
+    return { kind: 'call', fn: fnTok.value, args };
+  }
+
+  private parseArg(): Arg {
+    // Named arg: IDENT '=' expr (used by format).
+    const t = this.peek();
+    if (t?.type === 'ident' && this.tokens[this.pos + 1]?.type === '=') {
+      const nameTok = this.next();
+      this.expect('=');
+      return { name: nameTok.value, value: this.parseExpr() };
+    }
+    return { value: this.parseExpr() };
+  }
+
+  private parseRefOrBare(): RefNode | BareNode {
+    const first = this.expect('ident');
+    const segments = [first.value];
+
+    while (this.peek()?.type === '.') {
+      this.next();
+      segments.push(this.expect('ident').value);
+    }
+
+    if (REF_ROOT_SET.has(segments[0]) && segments.length > 1) {
+      return {
+        kind: 'ref',
+        root: segments[0] as RefRoot,
+        path: segments.slice(1),
+      };
+    }
+
+    // A known root with no path (`secret`) is meaningless; treat as bare so
+    // the evaluator can produce a clear "unknown identifier" style error in
+    // context rather than silently accepting it.
+    return { kind: 'bare', name: segments.join('.') };
+  }
+}
+
+/**
+ * Parse a computed-variable `value:` expression into an AST.
+ * Throws {@link ComputedParseError} on malformed input.
+ */
+export function parseComputed(expression: string): Node {
+  const tokens = tokenize(expression);
+  if (tokens.length === 0) {
+    throw new ComputedParseError('empty expression', 0);
+  }
+  return new Parser(tokens).parse();
+}

package/src/variables/computed/provider-lookup.ts

@@ -0,0 +1,130 @@
+/**
+ * Builds a {@link LookupFn} for evaluating a computed capability field in its
+ * PROVIDER's context.
+ *
+ * A computed field's DSL refs (`secret.ddns_passwords`, `self.hostname`,
+ * `system.primary_domain`, …) name values in the *provider* module's
+ * namespace — not the consumer that reads
+ * `$capability:dns_registrar.domain_list`. So when the resolver hits a
+ * computed marker it builds this lookup over the provider and evaluates there
+ * (the operator-chosen "evaluate in provider context, lazily" model).
+ *
+ * Values are returned TYPED — `module_configs.value_json` and JSON-encoded
+ * secrets are parsed back into objects/arrays — so DSL functions like
+ * `keys(secret.ddns_passwords)` operate on a real map, not a string.
+ *
+ * The lookup is built eagerly (it must `await getOrCreateMasterKey()` to
+ * decrypt secrets, and {@link LookupFn} is synchronous), then returned as a
+ * sync closure over the loaded namespaces.
+ */
+
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { moduleConfigs, secrets, systemConfig, systemSecrets } from '../../db/schema';
+import { decryptSecret } from '../../secrets/encryption';
+import { getOrCreateMasterKey } from '../../secrets/master-key';
+import { parseStoredConfigValue } from '../../services/module-config';
+import type { LookupFn } from './evaluate';
+
+/** Navigate a dotted path into a value; undefined if any segment is absent. */
+function navigate(root: unknown, path: string[]): unknown {
+  let current = root;
+  for (const seg of path) {
+    if (current && typeof current === 'object' && seg in (current as Record<string, unknown>)) {
+      current = (current as Record<string, unknown>)[seg];
+    } else {
+      return undefined;
+    }
+  }
+  return current;
+}
+
+/** Best-effort recover a typed value from a stored string (JSON or scalar). */
+function parseMaybeJson(raw: string): unknown {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return raw;
+  }
+}
+
+/**
+ * Build a lookup over `providerModuleId`'s config, secrets, and system data.
+ * Eagerly loads every namespace (decrypting secrets with the master key) and
+ * returns a synchronous lookup closure.
+ */
+export async function buildProviderLookup(
+  providerModuleId: string,
+  db: DbClient,
+): Promise<LookupFn> {
+  const masterKey = await getOrCreateMasterKey();
+
+  // self: the provider module's config, typed via value_json.
+  const selfMap: Record<string, unknown> = {};
+  for (const row of db
+    .select()
+    .from(moduleConfigs)
+    .where(eq(moduleConfigs.moduleId, providerModuleId))
+    .all()) {
+    try {
+      selfMap[row.key] = parseStoredConfigValue(row);
+    } catch {
+      // Pre-Defect-1 / unparseable row — fall back to the raw display value.
+      selfMap[row.key] = row.value;
+    }
+  }
+
+  // secret: the provider module's decrypted secrets, JSON-parsed where possible.
+  const secretMap: Record<string, unknown> = {};
+  for (const row of db.select().from(secrets).where(eq(secrets.moduleId, providerModuleId)).all()) {
+    try {
+      const plaintext = decryptSecret(
+        { encryptedValue: row.encryptedValue, iv: row.iv, authTag: row.authTag },
+        masterKey,
+      );
+      secretMap[row.name] = parseMaybeJson(plaintext);
+    } catch {
+      // Skip undecryptable secrets — a reference surfaces as "could not be
+      // resolved" from the evaluator.
+    }
+  }
+
+  // system: global config.
+  const systemMap: Record<string, unknown> = {};
+  for (const row of db.select().from(systemConfig).all()) {
+    systemMap[row.key] = parseMaybeJson(row.value);
+  }
+
+  // system_secret: decrypted global secrets.
+  const systemSecretMap: Record<string, unknown> = {};
+  for (const row of db.select().from(systemSecrets).all()) {
+    try {
+      const plaintext = decryptSecret(
+        { encryptedValue: row.encryptedValue, iv: row.iv, authTag: row.authTag },
+        masterKey,
+      );
+      systemSecretMap[row.key] = parseMaybeJson(plaintext);
+    } catch {
+      // skip undecryptable
+    }
+  }
+
+  return (root, path) => {
+    switch (root) {
+      case 'self':
+        return navigate(selfMap, path);
+      case 'secret':
+        return navigate(secretMap, path);
+      case 'system':
+        return navigate(systemMap, path);
+      case 'system_secret':
+        return navigate(systemSecretMap, path);
+      default:
+        // `capability` cross-refs from inside a computed field are not
+        // supported in v1 — they'd reintroduce the provider→provider coupling
+        // computed variables exist to avoid. Returns undefined → a clear
+        // "could not be resolved" error.
+        return undefined;
+    }
+  };
+}

package/src/variables/context.test.ts

@@ -5,8 +5,11 @@ import { eq } from 'drizzle-orm';
 import { type DbClient, createDbClient } from '../db/client';
 import {
   capabilities,
+  containerServices,
   ipAllocations,
   moduleConfigs,
+  moduleInfrastructure,
+  moduleSystems,
   modules,
   secrets,
   systemConfig,
@@ -15,6 +18,38 @@ import { buildContextFromData, buildResolutionContext } from './context';
 
 const TEST_DB_PATH = './test-context.db';
 
+/**
+ * Select a proxmox container_service for a module + give it a hostname, so the
+ * deployed-system recording in buildResolutionContext fires (IPAM allocates and
+ * records into module_systems). v2/MODULE_SYSTEMS_ADDRESSING.md.
+ */
+function setupProxmoxInfra(db: DbClient, moduleId: string, zone: string): void {
+  const serviceId = `svc-${moduleId}`;
+  db.insert(containerServices)
+    .values({
+      id: serviceId,
+      serviceId,
+      name: 'Test Proxmox',
+      providerName: 'proxmox',
+      zones: [zone] as Array<'internal' | 'dmz' | 'app' | 'secure' | 'external'>,
+      apiCredentialsEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
+      providerConfig: { default_target_node: 'pve', lxc_template: 't', storage: 's' },
+      verified: true,
+    })
+    .run();
+  db.insert(moduleInfrastructure)
+    .values({
+      id: `infra-${moduleId}`,
+      moduleId,
+      infrastructureType: 'container_service',
+      serviceId,
+    })
+    .run();
+  db.insert(moduleConfigs)
+    .values({ moduleId, key: 'hostname', value: moduleId, valueJson: JSON.stringify(moduleId) })
+    .run();
+}
+
 describe('Variable Context', () => {
   let db: DbClient;
 
@@ -444,30 +479,36 @@ describe('Variable Context', () => {
           },
         })
         .run();
+      setupProxmoxInfra(db, 'auto-module', 'dmz');
 
-      const context = await buildResolutionContext('auto-module', db);
-
-      // Should have auto-allocated vmid and target_ip
-      expect(context.selfConfig.vmid).toBe('2100');
-      expect(context.selfConfig.target_ip).toBe('10.0.10.10/24');
+      await buildResolutionContext('auto-module', db);
 
-      // Verify allocation persisted to database
-      const allocations = await db.select().from(ipAllocations).all();
+      // IPAM allocation persisted
+      const allocations = db.select().from(ipAllocations).all();
       expect(allocations).toHaveLength(1);
       expect(allocations[0].moduleId).toBe('auto-module');
       expect(allocations[0].vmid).toBe(2100);
       expect(allocations[0].containerIp).toBe('10.0.10.10/24');
 
-      // Verify config persisted to database
-      const configs = await db
+      // The system is recorded in module_systems (target_ip no longer in config).
+      const systems = db
+        .select()
+        .from(moduleSystems)
+        .where(eq(moduleSystems.moduleId, 'auto-module'))
+        .all();
+      expect(systems).toHaveLength(1);
+      expect(systems[0].name).toBe('main');
+      expect(systems[0].vmid).toBe(2100);
+      expect(systems[0].ipv4Address).toBe('10.0.10.10'); // CIDR-stripped
+
+      // target_ip / vmid are NOT written to module_configs anymore.
+      const configs = db
         .select()
         .from(moduleConfigs)
         .where(eq(moduleConfigs.moduleId, 'auto-module'))
         .all();
-      const vmidConfig = configs.find((c) => c.key === 'vmid');
-      const ipConfig = configs.find((c) => c.key === 'target_ip');
-      expect(vmidConfig?.value).toBe('2100');
-      expect(ipConfig?.value).toBe('10.0.10.10/24');
+      expect(configs.find((c) => c.key === 'target_ip')).toBeUndefined();
+      expect(configs.find((c) => c.key === 'vmid')).toBeUndefined();
     });
 
     test('should reuse existing allocation if already allocated', async () => {
@@ -488,9 +529,11 @@ describe('Variable Context', () => {
                 { name: 'target_ip', type: 'string', required: true, source: 'user' },
               ],
             },
+            requires: { machine: { zone: 'dmz' } },
           },
         })
         .run();
+      setupProxmoxInfra(db, 'existing-alloc', 'dmz');
 
       // Pre-create allocation
       db.insert(ipAllocations)
@@ -502,14 +545,20 @@ describe('Variable Context', () => {
         })
         .run();
 
-      const context = await buildResolutionContext('existing-alloc', db);
+      await buildResolutionContext('existing-alloc', db);
 
-      // Should reuse existing allocation
-      expect(context.selfConfig.vmid).toBe('2150');
-      expect(context.selfConfig.target_ip).toBe('10.0.10.50/24');
+      // Should reuse the existing allocation when recording the system.
+      const systems = db
+        .select()
+        .from(moduleSystems)
+        .where(eq(moduleSystems.moduleId, 'existing-alloc'))
+        .all();
+      expect(systems).toHaveLength(1);
+      expect(systems[0].vmid).toBe(2150);
+      expect(systems[0].ipv4Address).toBe('10.0.10.50');
 
       // Should not create duplicate allocation
-      const allocations = await db.select().from(ipAllocations).all();
+      const allocations = db.select().from(ipAllocations).all();
       expect(allocations).toHaveLength(1);
     });
 
@@ -600,9 +649,11 @@ describe('Variable Context', () => {
                 { name: 'target_ip', type: 'string', required: true, source: 'user' },
               ],
             },
+            requires: { machine: { zone: 'dmz' } },
           },
         })
         .run();
+      setupProxmoxInfra(db, 'module1', 'dmz');
 
       // Create second module
       db.insert(modules)
@@ -621,23 +672,33 @@ describe('Variable Context', () => {
                 { name: 'target_ip', type: 'string', required: true, source: 'user' },
               ],
             },
+            requires: { machine: { zone: 'dmz' } },
           },
         })
         .run();
+      setupProxmoxInfra(db, 'module2', 'dmz');
 
       // Allocate for both
-      const context1 = await buildResolutionContext('module1', db);
-      const context2 = await buildResolutionContext('module2', db);
+      await buildResolutionContext('module1', db);
+      await buildResolutionContext('module2', db);
 
-      // Should have sequential VMIDs
-      expect(context1.selfConfig.vmid).toBe('2100');
-      expect(context2.selfConfig.vmid).toBe('2101');
-
-      // Should have sequential IPs
-      expect(context1.selfConfig.target_ip).toBe('10.0.10.10/24');
-      expect(context2.selfConfig.target_ip).toBe('10.0.10.11/24');
+      // Should have recorded systems with sequential VMIDs + IPs.
+      const sys1 = db
+        .select()
+        .from(moduleSystems)
+        .where(eq(moduleSystems.moduleId, 'module1'))
+        .all();
+      const sys2 = db
+        .select()
+        .from(moduleSystems)
+        .where(eq(moduleSystems.moduleId, 'module2'))
+        .all();
+      expect(sys1[0].vmid).toBe(2100);
+      expect(sys2[0].vmid).toBe(2101);
+      expect(sys1[0].ipv4Address).toBe('10.0.10.10');
+      expect(sys2[0].ipv4Address).toBe('10.0.10.11');
 
-      const allocations = await db.select().from(ipAllocations).all();
+      const allocations = db.select().from(ipAllocations).all();
       expect(allocations).toHaveLength(2);
     });